From 7485ba6b87993f8798aeeab2a1df6f1e9a815691 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 12:00:44 -0600 Subject: Build the Weft static site generator --- main.go | 696 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 696 insertions(+) create mode 100644 main.go (limited to 'main.go') diff --git a/main.go b/main.go new file mode 100644 index 0000000..412ba84 --- /dev/null +++ b/main.go @@ -0,0 +1,696 @@ +package main + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "html/template" + "io" + "io/fs" + "net/url" + "os" + "path" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + gmtext "github.com/yuin/goldmark/text" + "golang.org/x/net/html" + "gopkg.in/yaml.v3" +) + +const manifestName = ".weft-generated.json" + +type Page struct { + SourcePath string + OutputPath string + URL string + CanonicalURL string + Title string + Meta map[string]any + ModTime time.Time + Sections []template.HTML +} + +type templateData struct { + *Page + Pages map[string][]*Page +} + +type source struct { + path string + output string + kind string + page *Page +} + +type site struct { + root string + pages map[string][]*Page + sources []source + templates *template.Template + warnings []string +} + +type manifest struct { + Outputs []string `json:"outputs"` +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: weft ") + os.Exit(2) + } + if err := build(os.Args[1]); err != nil { + fmt.Fprintln(os.Stderr, "weft:", err) + os.Exit(1) + } +} + +func build(root string) error { + abs, err := filepath.Abs(root) + if err != nil { + return err + } + st, err := os.Stat(abs) + if err != nil { + return err + } + if !st.IsDir() { + return fmt.Errorf("site root is not a directory: %s", abs) + } + + s := &site{root: abs, pages: map[string][]*Page{}} + if err := s.discover(); err != nil { + return err + } + if err := s.loadTemplates(); err != nil { + return err + } + outputs, err := s.render() + if err != nil { + return err + } + if err := s.validate(outputs); err != nil { + return err + } + if err := s.write(outputs); err != nil { + return err + } + for _, warning := range s.warnings { + fmt.Fprintln(os.Stderr, "weft: warning:", warning) + } + return nil +} + +func excluded(rel string) bool { + for _, part := range strings.Split(filepath.ToSlash(rel), "/") { + if strings.HasPrefix(part, "_") || part == ".git" || part == ".jj" || strings.HasPrefix(part, ".weft-txn-") { + return true + } + } + return false +} + +func (s *site) discover() error { + outputs := map[string]string{} + var templatePaths []string + err := filepath.WalkDir(s.root, func(full string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + rel, err := filepath.Rel(s.root, full) + if err != nil || rel == "." { + return err + } + if excluded(rel) { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.IsDir() { + return nil + } + rel = filepath.ToSlash(rel) + info, err := entry.Info() + if err != nil { + return err + } + switch { + case strings.HasSuffix(rel, ".md"): + out := strings.TrimSuffix(rel, ".md") + ".html" + if err := claim(outputs, out, rel); err != nil { + return err + } + page, warnings, err := s.parsePage(rel, out, info) + if err != nil { + return fmt.Errorf("parse %s: %w", rel, err) + } + s.warnings = append(s.warnings, warnings...) + s.sources = append(s.sources, source{path: rel, output: out, kind: "markdown", page: page}) + s.addPage(page) + case strings.HasSuffix(rel, ".tmpl"): + templatePaths = append(templatePaths, rel) + var out string + if strings.HasSuffix(rel, ".html.tmpl") || strings.HasSuffix(rel, ".xml.tmpl") { + out = strings.TrimSuffix(rel, ".tmpl") + if err := claim(outputs, out, rel); err != nil { + return err + } + s.sources = append(s.sources, source{path: rel, output: out, kind: "template"}) + } + } + return nil + }) + if err != nil { + return err + } + slices.Sort(templatePaths) + // Keep all template paths as non-output sources so definitions are loaded. + for _, p := range templatePaths { + found := false + for _, src := range s.sources { + if src.path == p { + found = true + break + } + } + if !found { + s.sources = append(s.sources, source{path: p, kind: "partial"}) + } + } + sort.SliceStable(s.sources, func(i, j int) bool { return s.sources[i].path < s.sources[j].path }) + return nil +} + +func claim(outputs map[string]string, output, input string) error { + if previous, ok := outputs[output]; ok { + return fmt.Errorf("output collision: %s and %s both target %s", previous, input, output) + } + outputs[output] = input + return nil +} + +func (s *site) parsePage(rel, out string, info fs.FileInfo) (*Page, []string, error) { + raw, err := os.ReadFile(filepath.Join(s.root, filepath.FromSlash(rel))) + if err != nil { + return nil, nil, err + } + meta, body, err := frontmatter(raw) + if err != nil { + return nil, nil, err + } + md := goldmark.New(goldmark.WithExtensions(extension.GFM), goldmark.WithParserOptions(parser.WithAutoHeadingID())) + doc := md.Parser().Parse(gmtext.NewReader(body)) + rewriteLinks(doc) + title := extractTitle(doc, body) + var warnings []string + if title == "" { + if value, ok := meta["title"].(string); ok && strings.TrimSpace(value) != "" { + title = strings.TrimSpace(value) + } else { + base := strings.TrimSuffix(path.Base(rel), path.Ext(rel)) + title = strings.Title(strings.ReplaceAll(strings.ReplaceAll(base, "_", " "), "-", " ")) //nolint:staticcheck + warnings = append(warnings, fmt.Sprintf("%s has no H1; using filename-derived title %q", rel, title)) + } + } + sections, err := renderSections(md, doc, body) + if err != nil { + return nil, nil, err + } + urlPath := "/" + out + if path.Base(out) == "index.html" { + urlPath = "/" + strings.TrimSuffix(out, "index.html") + } + return &Page{ + SourcePath: rel, OutputPath: out, URL: urlPath, + CanonicalURL: "https://tjp.lol" + urlPath, Title: title, Meta: meta, + ModTime: info.ModTime(), Sections: sections, + }, warnings, nil +} + +func frontmatter(raw []byte) (map[string]any, []byte, error) { + meta := map[string]any{} + if !bytes.HasPrefix(raw, []byte("---\n")) && !bytes.HasPrefix(raw, []byte("---\r\n")) { + return meta, raw, nil + } + lines := bytes.SplitAfter(raw, []byte("\n")) + end, offset := -1, len(lines[0]) + for i := 1; i < len(lines); i++ { + if strings.TrimSpace(string(lines[i])) == "---" { + end = i + break + } + offset += len(lines[i]) + } + if end < 0 { + return nil, nil, errors.New("unterminated YAML frontmatter") + } + front := bytes.Join(lines[1:end], nil) + if err := yaml.Unmarshal(front, &meta); err != nil { + return nil, nil, fmt.Errorf("YAML frontmatter: %w", err) + } + offset += len(lines[end]) + return meta, raw[offset:], nil +} + +func extractTitle(doc ast.Node, source []byte) string { + for node := doc.FirstChild(); node != nil; node = node.NextSibling() { + heading, ok := node.(*ast.Heading) + if !ok || heading.Level != 1 { + continue + } + title := strings.TrimSpace(string(heading.Text(source))) + doc.RemoveChild(doc, node) + return title + } + return "" +} + +func rewriteLinks(root ast.Node) { + _ = ast.Walk(root, func(node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + if link, ok := node.(*ast.Link); ok { + link.Destination = rewriteDestination(link.Destination) + } + return ast.WalkContinue, nil + }) +} + +func rewriteDestination(destination []byte) []byte { + raw := string(destination) + u, err := url.Parse(raw) + if err != nil || u.Scheme != "" || u.Host != "" || strings.HasPrefix(raw, "#") || !strings.HasSuffix(strings.ToLower(u.Path), ".md") { + return destination + } + u.Path = u.Path[:len(u.Path)-3] + ".html" + return []byte(u.String()) +} + +func renderSections(md goldmark.Markdown, doc ast.Node, source []byte) ([]template.HTML, error) { + var sections []template.HTML + for node := doc.FirstChild(); node != nil; { + next := node.NextSibling() + doc.RemoveChild(doc, node) + fragment := ast.NewDocument() + fragment.AppendChild(fragment, node) + var out bytes.Buffer + if err := md.Renderer().Render(&out, source, fragment); err != nil { + return nil, err + } + sections = append(sections, template.HTML(out.String())) // Goldmark raw HTML is disabled. + node = next + } + return sections, nil +} + +func (s *site) addPage(page *Page) { + dir := path.Dir(page.OutputPath) + if dir == "." { + dir = "" + } + for { + s.pages[dir] = append(s.pages[dir], page) + if dir == "" { + break + } + dir = path.Dir(dir) + if dir == "." { + dir = "" + } + } +} + +func (s *site) loadTemplates() error { + t := template.New("weft").Funcs(templateFuncs()) + for _, src := range s.sources { + if src.kind != "template" && src.kind != "partial" { + continue + } + contents, err := os.ReadFile(filepath.Join(s.root, filepath.FromSlash(src.path))) + if err != nil { + return err + } + if _, err := t.New(src.path).Parse(string(contents)); err != nil { + return fmt.Errorf("parse template %s: %w", src.path, err) + } + } + s.templates = t + return nil +} + +func templateFuncs() template.FuncMap { + return template.FuncMap{ + "rel": relativePath, + "sortPages": sortPages, + "filterPages": filterPages, + "slicePages": slicePages, + "joinSections": joinSections, + "xml": func(value any) string { return fmt.Sprint(value) }, + "date": func(value any, layout string) string { t, _ := asTime(value); return t.Format(layout) }, + "rfc3339": func(value any) string { t, _ := asTime(value); return t.Format(time.RFC3339) }, + } +} + +func relativePath(from, target string) string { + if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") || strings.HasPrefix(target, "mailto:") || strings.HasPrefix(target, "#") { + return target + } + fromDir := path.Dir(strings.TrimPrefix(from, "/")) + if fromDir == "." { + fromDir = "" + } + target = strings.TrimPrefix(target, "/") + rel, err := filepath.Rel(filepath.FromSlash(fromDir), filepath.FromSlash(target)) + if err != nil { + return target + } + return filepath.ToSlash(rel) +} + +func sortPages(pages []*Page, key string) []*Page { + result := slices.Clone(pages) + sort.SliceStable(result, func(i, j int) bool { + if key == "mod_time" { + return result[i].ModTime.After(result[j].ModTime) + } + if key == "title" { + return strings.ToLower(result[i].Title) < strings.ToLower(result[j].Title) + } + a, _ := asTime(result[i].Meta[key]) + b, _ := asTime(result[j].Meta[key]) + return a.After(b) + }) + return result +} + +func filterPages(pages []*Page, key string, value any) []*Page { + var result []*Page + for _, page := range pages { + var actual any + switch key { + case "title": + actual = page.Title + case "output": + actual = page.OutputPath + default: + actual = page.Meta[key] + } + if fmt.Sprint(actual) == fmt.Sprint(value) { + result = append(result, page) + } + } + return result +} + +func slicePages(pages []*Page, start, end int) []*Page { + if start < 0 { + start = 0 + } + if start > len(pages) { + start = len(pages) + } + if end < start { + end = start + } + if end > len(pages) { + end = len(pages) + } + return pages[start:end] +} + +func joinSections(sections []template.HTML) template.HTML { + var out strings.Builder + for _, section := range sections { + out.WriteString(string(section)) + } + return template.HTML(out.String()) +} + +func asTime(value any) (time.Time, bool) { + switch v := value.(type) { + case time.Time: + return v, true + case string: + for _, layout := range []string{time.RFC3339, "2006-01-02"} { + if parsed, err := time.Parse(layout, v); err == nil { + return parsed, true + } + } + } + return time.Time{}, false +} + +func (s *site) render() (map[string][]byte, error) { + outputs := map[string][]byte{} + for _, src := range s.sources { + if src.output == "" { + continue + } + var current *Page + name := src.path + if src.kind == "markdown" { + current = src.page + name = "page" + if src.output == "weblog" || strings.HasPrefix(src.output, "weblog/") { + name = "weblog" + } + if src.output == "notes" || strings.HasPrefix(src.output, "notes/") { + name = "note" + } + } else { + current = &Page{SourcePath: src.path, OutputPath: src.output, URL: "/" + src.output, CanonicalURL: "https://tjp.lol/" + src.output, Meta: map[string]any{}} + } + if s.templates.Lookup(name) == nil { + return nil, fmt.Errorf("%s requires missing template %q", src.path, name) + } + var out bytes.Buffer + if err := s.templates.ExecuteTemplate(&out, name, templateData{Page: current, Pages: s.pages}); err != nil { + return nil, fmt.Errorf("render %s: %w", src.path, err) + } + outputs[src.output] = out.Bytes() + } + return outputs, nil +} + +func (s *site) validate(outputs map[string][]byte) error { + tracked, err := s.readManifest() + if err != nil { + return err + } + for output, contents := range outputs { + if strings.HasSuffix(output, ".xml") { + var value struct{ XMLName xml.Name } + if err := xml.Unmarshal(contents, &value); err != nil { + return fmt.Errorf("invalid XML in %s: %w", output, err) + } + if output == "feed.xml" && (value.XMLName.Local != "feed" || value.XMLName.Space != "http://www.w3.org/2005/Atom") { + return fmt.Errorf("feed.xml is not an Atom 1.0 feed") + } + } + if strings.HasSuffix(output, ".html") { + if err := s.validateHTMLLinks(output, contents, outputs, tracked); err != nil { + return err + } + } + } + return nil +} + +func (s *site) validateHTMLLinks(output string, contents []byte, outputs map[string][]byte, tracked map[string]bool) error { + z := html.NewTokenizer(bytes.NewReader(contents)) + for { + tokenType := z.Next() + if tokenType == html.ErrorToken { + if errors.Is(z.Err(), io.EOF) { + return nil + } + return fmt.Errorf("parse HTML %s: %w", output, z.Err()) + } + if tokenType != html.StartTagToken && tokenType != html.SelfClosingTagToken { + continue + } + token := z.Token() + if token.Data != "a" { + continue + } + for _, attr := range token.Attr { + if attr.Key != "href" { + continue + } + if err := s.validateLink(output, attr.Val, outputs, tracked); err != nil { + return err + } + } + } +} + +func (s *site) validateLink(from, raw string, outputs map[string][]byte, tracked map[string]bool) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("invalid link in %s: %q", from, raw) + } + if u.Scheme != "" || u.Host != "" || u.Path == "" { + return nil + } + target := u.Path + if strings.HasPrefix(target, "/") { + target = strings.TrimPrefix(path.Clean(target), "/") + } else { + target = path.Clean(path.Join(path.Dir(from), target)) + } + if strings.HasSuffix(u.Path, "/") { + target = path.Join(target, "index.html") + } + if target == ".." || strings.HasPrefix(target, "../") { + return fmt.Errorf("internal link escapes the site root in %s: %s", from, raw) + } + if _, ok := outputs[target]; ok { + return nil + } + full := filepath.Join(s.root, filepath.FromSlash(target)) + if info, statErr := os.Stat(full); statErr == nil && !info.IsDir() && !tracked[target] { + return nil + } + indexTarget := path.Join(target, "index.html") + if info, statErr := os.Stat(filepath.Join(full, "index.html")); statErr == nil && !info.IsDir() && !tracked[indexTarget] { + return nil + } + return fmt.Errorf("broken internal link in %s: %s", from, raw) +} + +func (s *site) readManifest() (map[string]bool, error) { + tracked := map[string]bool{} + raw, err := os.ReadFile(filepath.Join(s.root, manifestName)) + if errors.Is(err, os.ErrNotExist) { + return tracked, nil + } + if err != nil { + return nil, err + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("read %s: %w", manifestName, err) + } + for _, output := range m.Outputs { + clean := path.Clean(output) + ext := path.Ext(clean) + if output == "" || output != clean || strings.Contains(output, `\`) || path.IsAbs(output) || strings.HasPrefix(clean, "../") || ext != ".html" && ext != ".xml" { + return nil, fmt.Errorf("unsafe path in %s: %q", manifestName, output) + } + tracked[output] = true + } + return tracked, nil +} + +func (s *site) write(outputs map[string][]byte) error { + tracked, err := s.readManifest() + if err != nil { + return err + } + for output := range outputs { + _, statErr := os.Stat(filepath.Join(s.root, filepath.FromSlash(output))) + if statErr == nil && !tracked[output] { + return fmt.Errorf("refusing to overwrite untracked output %s", output) + } + if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + return statErr + } + } + + txn, err := os.MkdirTemp(s.root, ".weft-txn-") + if err != nil { + return err + } + defer os.RemoveAll(txn) + stage := filepath.Join(txn, "stage") + backup := filepath.Join(txn, "backup") + if err := os.MkdirAll(stage, 0o755); err != nil { + return err + } + for output, contents := range outputs { + name := filepath.Join(stage, filepath.FromSlash(output)) + if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { + return err + } + if err := os.WriteFile(name, contents, 0o644); err != nil { + return err + } + } + paths := make([]string, 0, len(outputs)) + for output := range outputs { + paths = append(paths, output) + } + slices.Sort(paths) + manifestBytes, _ := json.MarshalIndent(manifest{Outputs: paths}, "", " ") + manifestBytes = append(manifestBytes, '\n') + if err := os.WriteFile(filepath.Join(stage, manifestName), manifestBytes, 0o644); err != nil { + return err + } + + var movedOld, installed []string + rollback := func() { + for i := len(installed) - 1; i >= 0; i-- { + _ = os.Remove(filepath.Join(s.root, filepath.FromSlash(installed[i]))) + } + for i := len(movedOld) - 1; i >= 0; i-- { + from := filepath.Join(backup, filepath.FromSlash(movedOld[i])) + to := filepath.Join(s.root, filepath.FromSlash(movedOld[i])) + _ = os.MkdirAll(filepath.Dir(to), 0o755) + _ = os.Rename(from, to) + } + } + moveOld := func(rel string) error { + from := filepath.Join(s.root, filepath.FromSlash(rel)) + if _, err := os.Stat(from); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + to := filepath.Join(backup, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil { + return err + } + if err := os.Rename(from, to); err != nil { + return err + } + movedOld = append(movedOld, rel) + return nil + } + for old := range tracked { + if _, wanted := outputs[old]; !wanted { + if err := moveOld(old); err != nil { + rollback() + return err + } + } + } + for _, rel := range append(paths, manifestName) { + if err := moveOld(rel); err != nil { + rollback() + return err + } + to := filepath.Join(s.root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil { + rollback() + return err + } + if err := os.Rename(filepath.Join(stage, filepath.FromSlash(rel)), to); err != nil { + rollback() + return err + } + installed = append(installed, rel) + } + return nil +} -- cgit v1.3