Build the Weft static site generator

1 files changed, 696 insertions, 0 deletions

diff --git a/main.go b/main.go
new file mode 100644
index 0000000..412ba84
--- /dev/null
+++ b/main.go
@@ -0,0 +1,696 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"html/template"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/ast"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	gmtext "github.com/yuin/goldmark/text"
+	"golang.org/x/net/html"
+	"gopkg.in/yaml.v3"
+)
+
+const manifestName = ".weft-generated.json"
+
+type Page struct {
+	SourcePath   string
+	OutputPath   string
+	URL          string
+	CanonicalURL string
+	Title        string
+	Meta         map[string]any
+	ModTime      time.Time
+	Sections     []template.HTML
+}
+
+type templateData struct {
+	*Page
+	Pages map[string][]*Page
+}
+
+type source struct {
+	path   string
+	output string
+	kind   string
+	page   *Page
+}
+
+type site struct {
+	root      string
+	pages     map[string][]*Page
+	sources   []source
+	templates *template.Template
+	warnings  []string
+}
+
+type manifest struct {
+	Outputs []string `json:"outputs"`
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintln(os.Stderr, "usage: weft <site-root>")
+		os.Exit(2)
+	}
+	if err := build(os.Args[1]); err != nil {
+		fmt.Fprintln(os.Stderr, "weft:", err)
+		os.Exit(1)
+	}
+}
+
+func build(root string) error {
+	abs, err := filepath.Abs(root)
+	if err != nil {
+		return err
+	}
+	st, err := os.Stat(abs)
+	if err != nil {
+		return err
+	}
+	if !st.IsDir() {
+		return fmt.Errorf("site root is not a directory: %s", abs)
+	}
+
+	s := &site{root: abs, pages: map[string][]*Page{}}
+	if err := s.discover(); err != nil {
+		return err
+	}
+	if err := s.loadTemplates(); err != nil {
+		return err
+	}
+	outputs, err := s.render()
+	if err != nil {
+		return err
+	}
+	if err := s.validate(outputs); err != nil {
+		return err
+	}
+	if err := s.write(outputs); err != nil {
+		return err
+	}
+	for _, warning := range s.warnings {
+		fmt.Fprintln(os.Stderr, "weft: warning:", warning)
+	}
+	return nil
+}
+
+func excluded(rel string) bool {
+	for _, part := range strings.Split(filepath.ToSlash(rel), "/") {
+		if strings.HasPrefix(part, "_") || part == ".git" || part == ".jj" || strings.HasPrefix(part, ".weft-txn-") {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *site) discover() error {
+	outputs := map[string]string{}
+	var templatePaths []string
+	err := filepath.WalkDir(s.root, func(full string, entry fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		rel, err := filepath.Rel(s.root, full)
+		if err != nil || rel == "." {
+			return err
+		}
+		if excluded(rel) {
+			if entry.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		rel = filepath.ToSlash(rel)
+		info, err := entry.Info()
+		if err != nil {
+			return err
+		}
+		switch {
+		case strings.HasSuffix(rel, ".md"):
+			out := strings.TrimSuffix(rel, ".md") + ".html"
+			if err := claim(outputs, out, rel); err != nil {
+				return err
+			}
+			page, warnings, err := s.parsePage(rel, out, info)
+			if err != nil {
+				return fmt.Errorf("parse %s: %w", rel, err)
+			}
+			s.warnings = append(s.warnings, warnings...)
+			s.sources = append(s.sources, source{path: rel, output: out, kind: "markdown", page: page})
+			s.addPage(page)
+		case strings.HasSuffix(rel, ".tmpl"):
+			templatePaths = append(templatePaths, rel)
+			var out string
+			if strings.HasSuffix(rel, ".html.tmpl") || strings.HasSuffix(rel, ".xml.tmpl") {
+				out = strings.TrimSuffix(rel, ".tmpl")
+				if err := claim(outputs, out, rel); err != nil {
+					return err
+				}
+				s.sources = append(s.sources, source{path: rel, output: out, kind: "template"})
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	slices.Sort(templatePaths)
+	// Keep all template paths as non-output sources so definitions are loaded.
+	for _, p := range templatePaths {
+		found := false
+		for _, src := range s.sources {
+			if src.path == p {
+				found = true
+				break
+			}
+		}
+		if !found {
+			s.sources = append(s.sources, source{path: p, kind: "partial"})
+		}
+	}
+	sort.SliceStable(s.sources, func(i, j int) bool { return s.sources[i].path < s.sources[j].path })
+	return nil
+}
+
+func claim(outputs map[string]string, output, input string) error {
+	if previous, ok := outputs[output]; ok {
+		return fmt.Errorf("output collision: %s and %s both target %s", previous, input, output)
+	}
+	outputs[output] = input
+	return nil
+}
+
+func (s *site) parsePage(rel, out string, info fs.FileInfo) (*Page, []string, error) {
+	raw, err := os.ReadFile(filepath.Join(s.root, filepath.FromSlash(rel)))
+	if err != nil {
+		return nil, nil, err
+	}
+	meta, body, err := frontmatter(raw)
+	if err != nil {
+		return nil, nil, err
+	}
+	md := goldmark.New(goldmark.WithExtensions(extension.GFM), goldmark.WithParserOptions(parser.WithAutoHeadingID()))
+	doc := md.Parser().Parse(gmtext.NewReader(body))
+	rewriteLinks(doc)
+	title := extractTitle(doc, body)
+	var warnings []string
+	if title == "" {
+		if value, ok := meta["title"].(string); ok && strings.TrimSpace(value) != "" {
+			title = strings.TrimSpace(value)
+		} else {
+			base := strings.TrimSuffix(path.Base(rel), path.Ext(rel))
+			title = strings.Title(strings.ReplaceAll(strings.ReplaceAll(base, "_", " "), "-", " ")) //nolint:staticcheck
+			warnings = append(warnings, fmt.Sprintf("%s has no H1; using filename-derived title %q", rel, title))
+		}
+	}
+	sections, err := renderSections(md, doc, body)
+	if err != nil {
+		return nil, nil, err
+	}
+	urlPath := "/" + out
+	if path.Base(out) == "index.html" {
+		urlPath = "/" + strings.TrimSuffix(out, "index.html")
+	}
+	return &Page{
+		SourcePath: rel, OutputPath: out, URL: urlPath,
+		CanonicalURL: "https://tjp.lol" + urlPath, Title: title, Meta: meta,
+		ModTime: info.ModTime(), Sections: sections,
+	}, warnings, nil
+}
+
+func frontmatter(raw []byte) (map[string]any, []byte, error) {
+	meta := map[string]any{}
+	if !bytes.HasPrefix(raw, []byte("---\n")) && !bytes.HasPrefix(raw, []byte("---\r\n")) {
+		return meta, raw, nil
+	}
+	lines := bytes.SplitAfter(raw, []byte("\n"))
+	end, offset := -1, len(lines[0])
+	for i := 1; i < len(lines); i++ {
+		if strings.TrimSpace(string(lines[i])) == "---" {
+			end = i
+			break
+		}
+		offset += len(lines[i])
+	}
+	if end < 0 {
+		return nil, nil, errors.New("unterminated YAML frontmatter")
+	}
+	front := bytes.Join(lines[1:end], nil)
+	if err := yaml.Unmarshal(front, &meta); err != nil {
+		return nil, nil, fmt.Errorf("YAML frontmatter: %w", err)
+	}
+	offset += len(lines[end])
+	return meta, raw[offset:], nil
+}
+
+func extractTitle(doc ast.Node, source []byte) string {
+	for node := doc.FirstChild(); node != nil; node = node.NextSibling() {
+		heading, ok := node.(*ast.Heading)
+		if !ok || heading.Level != 1 {
+			continue
+		}
+		title := strings.TrimSpace(string(heading.Text(source)))
+		doc.RemoveChild(doc, node)
+		return title
+	}
+	return ""
+}
+
+func rewriteLinks(root ast.Node) {
+	_ = ast.Walk(root, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
+		if !entering {
+			return ast.WalkContinue, nil
+		}
+		if link, ok := node.(*ast.Link); ok {
+			link.Destination = rewriteDestination(link.Destination)
+		}
+		return ast.WalkContinue, nil
+	})
+}
+
+func rewriteDestination(destination []byte) []byte {
+	raw := string(destination)
+	u, err := url.Parse(raw)
+	if err != nil || u.Scheme != "" || u.Host != "" || strings.HasPrefix(raw, "#") || !strings.HasSuffix(strings.ToLower(u.Path), ".md") {
+		return destination
+	}
+	u.Path = u.Path[:len(u.Path)-3] + ".html"
+	return []byte(u.String())
+}
+
+func renderSections(md goldmark.Markdown, doc ast.Node, source []byte) ([]template.HTML, error) {
+	var sections []template.HTML
+	for node := doc.FirstChild(); node != nil; {
+		next := node.NextSibling()
+		doc.RemoveChild(doc, node)
+		fragment := ast.NewDocument()
+		fragment.AppendChild(fragment, node)
+		var out bytes.Buffer
+		if err := md.Renderer().Render(&out, source, fragment); err != nil {
+			return nil, err
+		}
+		sections = append(sections, template.HTML(out.String())) // Goldmark raw HTML is disabled.
+		node = next
+	}
+	return sections, nil
+}
+
+func (s *site) addPage(page *Page) {
+	dir := path.Dir(page.OutputPath)
+	if dir == "." {
+		dir = ""
+	}
+	for {
+		s.pages[dir] = append(s.pages[dir], page)
+		if dir == "" {
+			break
+		}
+		dir = path.Dir(dir)
+		if dir == "." {
+			dir = ""
+		}
+	}
+}
+
+func (s *site) loadTemplates() error {
+	t := template.New("weft").Funcs(templateFuncs())
+	for _, src := range s.sources {
+		if src.kind != "template" && src.kind != "partial" {
+			continue
+		}
+		contents, err := os.ReadFile(filepath.Join(s.root, filepath.FromSlash(src.path)))
+		if err != nil {
+			return err
+		}
+		if _, err := t.New(src.path).Parse(string(contents)); err != nil {
+			return fmt.Errorf("parse template %s: %w", src.path, err)
+		}
+	}
+	s.templates = t
+	return nil
+}
+
+func templateFuncs() template.FuncMap {
+	return template.FuncMap{
+		"rel":          relativePath,
+		"sortPages":    sortPages,
+		"filterPages":  filterPages,
+		"slicePages":   slicePages,
+		"joinSections": joinSections,
+		"xml":          func(value any) string { return fmt.Sprint(value) },
+		"date":         func(value any, layout string) string { t, _ := asTime(value); return t.Format(layout) },
+		"rfc3339":      func(value any) string { t, _ := asTime(value); return t.Format(time.RFC3339) },
+	}
+}
+
+func relativePath(from, target string) string {
+	if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") || strings.HasPrefix(target, "mailto:") || strings.HasPrefix(target, "#") {
+		return target
+	}
+	fromDir := path.Dir(strings.TrimPrefix(from, "/"))
+	if fromDir == "." {
+		fromDir = ""
+	}
+	target = strings.TrimPrefix(target, "/")
+	rel, err := filepath.Rel(filepath.FromSlash(fromDir), filepath.FromSlash(target))
+	if err != nil {
+		return target
+	}
+	return filepath.ToSlash(rel)
+}
+
+func sortPages(pages []*Page, key string) []*Page {
+	result := slices.Clone(pages)
+	sort.SliceStable(result, func(i, j int) bool {
+		if key == "mod_time" {
+			return result[i].ModTime.After(result[j].ModTime)
+		}
+		if key == "title" {
+			return strings.ToLower(result[i].Title) < strings.ToLower(result[j].Title)
+		}
+		a, _ := asTime(result[i].Meta[key])
+		b, _ := asTime(result[j].Meta[key])
+		return a.After(b)
+	})
+	return result
+}
+
+func filterPages(pages []*Page, key string, value any) []*Page {
+	var result []*Page
+	for _, page := range pages {
+		var actual any
+		switch key {
+		case "title":
+			actual = page.Title
+		case "output":
+			actual = page.OutputPath
+		default:
+			actual = page.Meta[key]
+		}
+		if fmt.Sprint(actual) == fmt.Sprint(value) {
+			result = append(result, page)
+		}
+	}
+	return result
+}
+
+func slicePages(pages []*Page, start, end int) []*Page {
+	if start < 0 {
+		start = 0
+	}
+	if start > len(pages) {
+		start = len(pages)
+	}
+	if end < start {
+		end = start
+	}
+	if end > len(pages) {
+		end = len(pages)
+	}
+	return pages[start:end]
+}
+
+func joinSections(sections []template.HTML) template.HTML {
+	var out strings.Builder
+	for _, section := range sections {
+		out.WriteString(string(section))
+	}
+	return template.HTML(out.String())
+}
+
+func asTime(value any) (time.Time, bool) {
+	switch v := value.(type) {
+	case time.Time:
+		return v, true
+	case string:
+		for _, layout := range []string{time.RFC3339, "2006-01-02"} {
+			if parsed, err := time.Parse(layout, v); err == nil {
+				return parsed, true
+			}
+		}
+	}
+	return time.Time{}, false
+}
+
+func (s *site) render() (map[string][]byte, error) {
+	outputs := map[string][]byte{}
+	for _, src := range s.sources {
+		if src.output == "" {
+			continue
+		}
+		var current *Page
+		name := src.path
+		if src.kind == "markdown" {
+			current = src.page
+			name = "page"
+			if src.output == "weblog" || strings.HasPrefix(src.output, "weblog/") {
+				name = "weblog"
+			}
+			if src.output == "notes" || strings.HasPrefix(src.output, "notes/") {
+				name = "note"
+			}
+		} else {
+			current = &Page{SourcePath: src.path, OutputPath: src.output, URL: "/" + src.output, CanonicalURL: "https://tjp.lol/" + src.output, Meta: map[string]any{}}
+		}
+		if s.templates.Lookup(name) == nil {
+			return nil, fmt.Errorf("%s requires missing template %q", src.path, name)
+		}
+		var out bytes.Buffer
+		if err := s.templates.ExecuteTemplate(&out, name, templateData{Page: current, Pages: s.pages}); err != nil {
+			return nil, fmt.Errorf("render %s: %w", src.path, err)
+		}
+		outputs[src.output] = out.Bytes()
+	}
+	return outputs, nil
+}
+
+func (s *site) validate(outputs map[string][]byte) error {
+	tracked, err := s.readManifest()
+	if err != nil {
+		return err
+	}
+	for output, contents := range outputs {
+		if strings.HasSuffix(output, ".xml") {
+			var value struct{ XMLName xml.Name }
+			if err := xml.Unmarshal(contents, &value); err != nil {
+				return fmt.Errorf("invalid XML in %s: %w", output, err)
+			}
+			if output == "feed.xml" && (value.XMLName.Local != "feed" || value.XMLName.Space != "http://www.w3.org/2005/Atom") {
+				return fmt.Errorf("feed.xml is not an Atom 1.0 feed")
+			}
+		}
+		if strings.HasSuffix(output, ".html") {
+			if err := s.validateHTMLLinks(output, contents, outputs, tracked); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (s *site) validateHTMLLinks(output string, contents []byte, outputs map[string][]byte, tracked map[string]bool) error {
+	z := html.NewTokenizer(bytes.NewReader(contents))
+	for {
+		tokenType := z.Next()
+		if tokenType == html.ErrorToken {
+			if errors.Is(z.Err(), io.EOF) {
+				return nil
+			}
+			return fmt.Errorf("parse HTML %s: %w", output, z.Err())
+		}
+		if tokenType != html.StartTagToken && tokenType != html.SelfClosingTagToken {
+			continue
+		}
+		token := z.Token()
+		if token.Data != "a" {
+			continue
+		}
+		for _, attr := range token.Attr {
+			if attr.Key != "href" {
+				continue
+			}
+			if err := s.validateLink(output, attr.Val, outputs, tracked); err != nil {
+				return err
+			}
+		}
+	}
+}
+
+func (s *site) validateLink(from, raw string, outputs map[string][]byte, tracked map[string]bool) error {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return fmt.Errorf("invalid link in %s: %q", from, raw)
+	}
+	if u.Scheme != "" || u.Host != "" || u.Path == "" {
+		return nil
+	}
+	target := u.Path
+	if strings.HasPrefix(target, "/") {
+		target = strings.TrimPrefix(path.Clean(target), "/")
+	} else {
+		target = path.Clean(path.Join(path.Dir(from), target))
+	}
+	if strings.HasSuffix(u.Path, "/") {
+		target = path.Join(target, "index.html")
+	}
+	if target == ".." || strings.HasPrefix(target, "../") {
+		return fmt.Errorf("internal link escapes the site root in %s: %s", from, raw)
+	}
+	if _, ok := outputs[target]; ok {
+		return nil
+	}
+	full := filepath.Join(s.root, filepath.FromSlash(target))
+	if info, statErr := os.Stat(full); statErr == nil && !info.IsDir() && !tracked[target] {
+		return nil
+	}
+	indexTarget := path.Join(target, "index.html")
+	if info, statErr := os.Stat(filepath.Join(full, "index.html")); statErr == nil && !info.IsDir() && !tracked[indexTarget] {
+		return nil
+	}
+	return fmt.Errorf("broken internal link in %s: %s", from, raw)
+}
+
+func (s *site) readManifest() (map[string]bool, error) {
+	tracked := map[string]bool{}
+	raw, err := os.ReadFile(filepath.Join(s.root, manifestName))
+	if errors.Is(err, os.ErrNotExist) {
+		return tracked, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var m manifest
+	if err := json.Unmarshal(raw, &m); err != nil {
+		return nil, fmt.Errorf("read %s: %w", manifestName, err)
+	}
+	for _, output := range m.Outputs {
+		clean := path.Clean(output)
+		ext := path.Ext(clean)
+		if output == "" || output != clean || strings.Contains(output, `\`) || path.IsAbs(output) || strings.HasPrefix(clean, "../") || ext != ".html" && ext != ".xml" {
+			return nil, fmt.Errorf("unsafe path in %s: %q", manifestName, output)
+		}
+		tracked[output] = true
+	}
+	return tracked, nil
+}
+
+func (s *site) write(outputs map[string][]byte) error {
+	tracked, err := s.readManifest()
+	if err != nil {
+		return err
+	}
+	for output := range outputs {
+		_, statErr := os.Stat(filepath.Join(s.root, filepath.FromSlash(output)))
+		if statErr == nil && !tracked[output] {
+			return fmt.Errorf("refusing to overwrite untracked output %s", output)
+		}
+		if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {
+			return statErr
+		}
+	}
+
+	txn, err := os.MkdirTemp(s.root, ".weft-txn-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(txn)
+	stage := filepath.Join(txn, "stage")
+	backup := filepath.Join(txn, "backup")
+	if err := os.MkdirAll(stage, 0o755); err != nil {
+		return err
+	}
+	for output, contents := range outputs {
+		name := filepath.Join(stage, filepath.FromSlash(output))
+		if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
+			return err
+		}
+		if err := os.WriteFile(name, contents, 0o644); err != nil {
+			return err
+		}
+	}
+	paths := make([]string, 0, len(outputs))
+	for output := range outputs {
+		paths = append(paths, output)
+	}
+	slices.Sort(paths)
+	manifestBytes, _ := json.MarshalIndent(manifest{Outputs: paths}, "", "  ")
+	manifestBytes = append(manifestBytes, '\n')
+	if err := os.WriteFile(filepath.Join(stage, manifestName), manifestBytes, 0o644); err != nil {
+		return err
+	}
+
+	var movedOld, installed []string
+	rollback := func() {
+		for i := len(installed) - 1; i >= 0; i-- {
+			_ = os.Remove(filepath.Join(s.root, filepath.FromSlash(installed[i])))
+		}
+		for i := len(movedOld) - 1; i >= 0; i-- {
+			from := filepath.Join(backup, filepath.FromSlash(movedOld[i]))
+			to := filepath.Join(s.root, filepath.FromSlash(movedOld[i]))
+			_ = os.MkdirAll(filepath.Dir(to), 0o755)
+			_ = os.Rename(from, to)
+		}
+	}
+	moveOld := func(rel string) error {
+		from := filepath.Join(s.root, filepath.FromSlash(rel))
+		if _, err := os.Stat(from); errors.Is(err, os.ErrNotExist) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		to := filepath.Join(backup, filepath.FromSlash(rel))
+		if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil {
+			return err
+		}
+		if err := os.Rename(from, to); err != nil {
+			return err
+		}
+		movedOld = append(movedOld, rel)
+		return nil
+	}
+	for old := range tracked {
+		if _, wanted := outputs[old]; !wanted {
+			if err := moveOld(old); err != nil {
+				rollback()
+				return err
+			}
+		}
+	}
+	for _, rel := range append(paths, manifestName) {
+		if err := moveOld(rel); err != nil {
+			rollback()
+			return err
+		}
+		to := filepath.Join(s.root, filepath.FromSlash(rel))
+		if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil {
+			rollback()
+			return err
+		}
+		if err := os.Rename(filepath.Join(stage, filepath.FromSlash(rel)), to); err != nil {
+			rollback()
+			return err
+		}
+		installed = append(installed, rel)
+	}
+	return nil
+}