package main import ( "bytes" "encoding/json" "encoding/xml" "errors" "flag" "fmt" "html/template" "io" "io/fs" "net/url" "os" "path" "path/filepath" "slices" "sort" "strings" "syscall" "time" "github.com/yuin/goldmark" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" gmtext "github.com/yuin/goldmark/text" "golang.org/x/net/html" "gopkg.in/yaml.v3" ) const manifestName = ".weft-generated.json" const ( gitignoreStart = "# BEGIN weft generated outputs" gitignoreEnd = "# END weft generated outputs" ) type Page struct { SourcePath string OutputPath string URL string CanonicalURL string Title string Meta map[string]any ModTime time.Time Sections []template.HTML } type templateData struct { *Page Pages map[string][]*Page } type source struct { path string output string kind string page *Page } type site struct { root string canonicalRoot string pages map[string][]*Page sources []source templates *template.Template fallback string warnings []string } type manifest struct { Outputs []string `json:"outputs"` } func main() { flags := flag.NewFlagSet("weft", flag.ContinueOnError) fallback := flags.String("fallback", "page", "fallback template for Markdown pages") manageGitignore := flags.Bool("gitignore", false, "manage generated outputs in the site .gitignore") flags.Usage = func() { fmt.Fprintln(flags.Output(), "usage: weft [-gitignore] [-fallback template] ") } if err := flags.Parse(os.Args[1:]); err != nil { os.Exit(2) } if flags.NArg() != 2 { flags.Usage() os.Exit(2) } if err := buildWithFallback(flags.Arg(0), flags.Arg(1), *fallback, *manageGitignore); err != nil { fmt.Fprintln(os.Stderr, "weft:", err) os.Exit(1) } } func buildWithFallback(root, canonicalRoot, fallback string, manageGitignore bool) error { if strings.TrimSpace(fallback) == "" { return errors.New("fallback template cannot be empty") } abs, err := filepath.Abs(root) if err != nil { return err } st, err := os.Stat(abs) if err != nil { return err } if !st.IsDir() { return fmt.Errorf("site root is not a directory: %s", abs) } s := &site{root: abs, canonicalRoot: canonicalRoot, fallback: fallback, pages: map[string][]*Page{}} if err := s.discover(); err != nil { return err } if err := s.loadTemplates(); err != nil { return err } outputs, err := s.render() if err != nil { return err } if err := s.validate(outputs); err != nil { return err } if err := s.write(outputs, manageGitignore); err != nil { return err } for _, warning := range s.warnings { fmt.Fprintln(os.Stderr, "weft: warning:", warning) } return nil } func excluded(rel string) bool { for _, part := range strings.Split(filepath.ToSlash(rel), "/") { if strings.HasPrefix(part, "_") || strings.HasPrefix(part, ".") { return true } } return false } func (s *site) discover() error { outputs := map[string]string{} var templatePaths []string err := walkFiles(s.root, func(rel string, info fs.FileInfo) error { switch { case strings.HasSuffix(rel, ".md"): out := strings.TrimSuffix(rel, ".md") + ".html" if err := claim(outputs, out, rel); err != nil { return err } page, warnings, err := s.parsePage(rel, out, info) if err != nil { return fmt.Errorf("parse %s: %w", rel, err) } s.warnings = append(s.warnings, warnings...) s.sources = append(s.sources, source{path: rel, output: out, kind: "markdown", page: page}) s.addPage(page) case strings.HasSuffix(rel, ".tmpl"): templatePaths = append(templatePaths, rel) var out string if strings.HasSuffix(rel, ".html.tmpl") || strings.HasSuffix(rel, ".xml.tmpl") { out = strings.TrimSuffix(rel, ".tmpl") if err := claim(outputs, out, rel); err != nil { return err } s.sources = append(s.sources, source{path: rel, output: out, kind: "template"}) } } return nil }) if err != nil { return err } slices.Sort(templatePaths) // Keep all template paths as non-output sources so definitions are loaded. for _, p := range templatePaths { found := false for _, src := range s.sources { if src.path == p { found = true break } } if !found { s.sources = append(s.sources, source{path: p, kind: "partial"}) } } sort.SliceStable(s.sources, func(i, j int) bool { return s.sources[i].path < s.sources[j].path }) return nil } func walkFiles(root string, visit func(string, fs.FileInfo) error) error { active := map[string]bool{} var walk func(string, string) error walk = func(full, relDir string) error { real, err := filepath.EvalSymlinks(full) if err != nil { return err } if active[real] { return fmt.Errorf("symlink cycle at %s", filepath.ToSlash(relDir)) } active[real] = true defer delete(active, real) entries, err := os.ReadDir(full) if err != nil { return err } for _, entry := range entries { rel := filepath.Join(relDir, entry.Name()) if excluded(rel) { continue } name := filepath.Join(full, entry.Name()) info, err := os.Stat(name) if err != nil { return err } if info.IsDir() { if err := walk(name, rel); err != nil { return err } continue } if err := visit(filepath.ToSlash(rel), info); err != nil { return err } } return nil } return walk(root, "") } func claim(outputs map[string]string, output, input string) error { if previous, ok := outputs[output]; ok { return fmt.Errorf("output collision: %s and %s both target %s", previous, input, output) } outputs[output] = input return nil } func (s *site) parsePage(rel, out string, info fs.FileInfo) (*Page, []string, error) { raw, err := os.ReadFile(filepath.Join(s.root, filepath.FromSlash(rel))) if err != nil { return nil, nil, err } meta, body, err := frontmatter(raw) if err != nil { return nil, nil, err } md := goldmark.New(goldmark.WithExtensions(extension.GFM), goldmark.WithParserOptions(parser.WithAutoHeadingID())) doc := md.Parser().Parse(gmtext.NewReader(body)) rewriteLinks(doc) title := extractTitle(doc, body) var warnings []string if title == "" { if value, ok := meta["title"].(string); ok && strings.TrimSpace(value) != "" { title = strings.TrimSpace(value) } else { base := strings.TrimSuffix(path.Base(rel), path.Ext(rel)) title = strings.Title(strings.ReplaceAll(strings.ReplaceAll(base, "_", " "), "-", " ")) //nolint:staticcheck warnings = append(warnings, fmt.Sprintf("%s has no H1; using filename-derived title %q", rel, title)) } } sections, err := renderSections(md, doc, body) if err != nil { return nil, nil, err } urlPath := "/" + out if path.Base(out) == "index.html" { urlPath = "/" + strings.TrimSuffix(out, "index.html") } return &Page{ SourcePath: rel, OutputPath: out, URL: urlPath, CanonicalURL: s.canonicalRoot + urlPath, Title: title, Meta: meta, ModTime: info.ModTime(), Sections: sections, }, warnings, nil } func frontmatter(raw []byte) (map[string]any, []byte, error) { meta := map[string]any{} if !bytes.HasPrefix(raw, []byte("---\n")) && !bytes.HasPrefix(raw, []byte("---\r\n")) { return meta, raw, nil } lines := bytes.SplitAfter(raw, []byte("\n")) end, offset := -1, len(lines[0]) for i := 1; i < len(lines); i++ { if strings.TrimSpace(string(lines[i])) == "---" { end = i break } offset += len(lines[i]) } if end < 0 { return nil, nil, errors.New("unterminated YAML frontmatter") } front := bytes.Join(lines[1:end], nil) if err := yaml.Unmarshal(front, &meta); err != nil { return nil, nil, fmt.Errorf("YAML frontmatter: %w", err) } offset += len(lines[end]) return meta, raw[offset:], nil } func extractTitle(doc ast.Node, source []byte) string { for node := doc.FirstChild(); node != nil; node = node.NextSibling() { heading, ok := node.(*ast.Heading) if !ok || heading.Level != 1 { continue } return strings.TrimSpace(string(heading.Text(source))) } return "" } func rewriteLinks(root ast.Node) { _ = ast.Walk(root, func(node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } if link, ok := node.(*ast.Link); ok { link.Destination = rewriteDestination(link.Destination) } return ast.WalkContinue, nil }) } func rewriteDestination(destination []byte) []byte { raw := string(destination) u, err := url.Parse(raw) if err != nil || u.Scheme != "" || u.Host != "" || strings.HasPrefix(raw, "#") || !strings.HasSuffix(strings.ToLower(u.Path), ".md") { return destination } u.Path = u.Path[:len(u.Path)-3] + ".html" return []byte(u.String()) } func renderSections(md goldmark.Markdown, doc ast.Node, source []byte) ([]template.HTML, error) { var sections []template.HTML for node := doc.FirstChild(); node != nil; { next := node.NextSibling() doc.RemoveChild(doc, node) fragment := ast.NewDocument() fragment.AppendChild(fragment, node) var out bytes.Buffer if err := md.Renderer().Render(&out, source, fragment); err != nil { return nil, err } sections = append(sections, template.HTML(out.String())) // Goldmark raw HTML is disabled. node = next } return sections, nil } func (s *site) addPage(page *Page) { dir := path.Dir(page.OutputPath) if dir == "." { dir = "" } for { s.pages[dir] = append(s.pages[dir], page) if dir == "" { break } dir = path.Dir(dir) if dir == "." { dir = "" } } } func (s *site) loadTemplates() error { t := template.New("weft").Funcs(templateFuncs()) for _, src := range s.sources { if src.kind != "template" && src.kind != "partial" { continue } contents, err := os.ReadFile(filepath.Join(s.root, filepath.FromSlash(src.path))) if err != nil { return err } if _, err := t.New(src.path).Parse(string(contents)); err != nil { return fmt.Errorf("parse template %s: %w", src.path, err) } } s.templates = t return nil } func templateFuncs() template.FuncMap { return template.FuncMap{ "rel": relativePath, "sortPages": sortPages, "filterPages": filterPages, "slicePages": slicePages, "joinSections": joinSections, "xml": func(value any) string { return fmt.Sprint(value) }, "date": func(value any, layout string) string { t, _ := asTime(value); return t.Format(layout) }, "rfc3339": func(value any) string { t, _ := asTime(value); return t.Format(time.RFC3339) }, } } func relativePath(from, target string) string { if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") || strings.HasPrefix(target, "mailto:") || strings.HasPrefix(target, "#") { return target } directory := strings.HasSuffix(target, "/") fromDir := path.Dir(strings.TrimPrefix(from, "/")) if fromDir == "." { fromDir = "" } target = strings.TrimPrefix(target, "/") rel, err := filepath.Rel(filepath.FromSlash(fromDir), filepath.FromSlash(target)) if err != nil { return target } rel = path.Clean(filepath.ToSlash(rel)) if directory { return strings.TrimSuffix(rel, "/") + "/" } return rel } func sortPages(pages []*Page, keys ...string) []*Page { result := slices.Clone(pages) date := func(page *Page) time.Time { for _, key := range keys { if key == "mod_time" { return page.ModTime } if value, ok := asTime(page.Meta[key]); ok { return value } } return time.Time{} } sort.SliceStable(result, func(i, j int) bool { if len(keys) == 1 && keys[0] == "title" { return strings.ToLower(result[i].Title) < strings.ToLower(result[j].Title) } a := date(result[i]) b := date(result[j]) return a.After(b) }) return result } func filterPages(pages []*Page, key string, value any) []*Page { var result []*Page for _, page := range pages { var actual any switch key { case "title": actual = page.Title case "output": actual = page.OutputPath default: actual = page.Meta[key] } if fmt.Sprint(actual) == fmt.Sprint(value) { result = append(result, page) } } return result } func slicePages(pages []*Page, start, end int) []*Page { if start < 0 { start = 0 } if start > len(pages) { start = len(pages) } if end < start { end = start } if end > len(pages) { end = len(pages) } return pages[start:end] } func joinSections(sections []template.HTML) template.HTML { var out strings.Builder for _, section := range sections { out.WriteString(string(section)) } return template.HTML(out.String()) } func asTime(value any) (time.Time, bool) { switch v := value.(type) { case time.Time: return v, true case string: for _, layout := range []string{time.RFC3339, "2006-01-02"} { if parsed, err := time.Parse(layout, v); err == nil { return parsed, true } } } return time.Time{}, false } func (s *site) render() (map[string][]byte, error) { outputs := map[string][]byte{} for _, src := range s.sources { if src.output == "" { continue } var current *Page name := src.path if src.kind == "markdown" { current = src.page name = s.markdownTemplate(src.path) } else { current = &Page{SourcePath: src.path, OutputPath: src.output, URL: "/" + src.output, CanonicalURL: s.canonicalRoot + src.output, Meta: map[string]any{}} } if s.templates.Lookup(name) == nil { return nil, fmt.Errorf("%s requires missing template %q", src.path, name) } var out bytes.Buffer if err := s.templates.ExecuteTemplate(&out, name, templateData{Page: current, Pages: s.pages}); err != nil { return nil, fmt.Errorf("render %s: %w", src.path, err) } outputs[src.output] = out.Bytes() } return outputs, nil } func (s *site) markdownTemplate(sourcePath string) string { for candidate := strings.TrimSuffix(sourcePath, ".md"); candidate != "." && candidate != ""; candidate = path.Dir(candidate) { if s.templates.Lookup(candidate) != nil { return candidate } } return s.fallback } func (s *site) validate(outputs map[string][]byte) error { tracked, err := s.readManifest() if err != nil { return err } for output, contents := range outputs { if strings.HasSuffix(output, ".xml") { var value struct{ XMLName xml.Name } if err := xml.Unmarshal(contents, &value); err != nil { return fmt.Errorf("invalid XML in %s: %w", output, err) } if output == "feed.xml" && (value.XMLName.Local != "feed" || value.XMLName.Space != "http://www.w3.org/2005/Atom") { return fmt.Errorf("feed.xml is not an Atom 1.0 feed") } } if strings.HasSuffix(output, ".html") { if err := s.validateHTMLLinks(output, contents, outputs, tracked); err != nil { return err } } } return nil } func (s *site) validateHTMLLinks(output string, contents []byte, outputs map[string][]byte, tracked map[string]bool) error { z := html.NewTokenizer(bytes.NewReader(contents)) for { tokenType := z.Next() if tokenType == html.ErrorToken { if errors.Is(z.Err(), io.EOF) { return nil } return fmt.Errorf("parse HTML %s: %w", output, z.Err()) } if tokenType != html.StartTagToken && tokenType != html.SelfClosingTagToken { continue } token := z.Token() if token.Data != "a" { continue } for _, attr := range token.Attr { if attr.Key != "href" { continue } if err := s.validateLink(output, attr.Val, outputs, tracked); err != nil { return err } } } } func (s *site) validateLink(from, raw string, outputs map[string][]byte, tracked map[string]bool) error { u, err := url.Parse(raw) if err != nil { return fmt.Errorf("invalid link in %s: %q", from, raw) } if u.Scheme != "" || u.Host != "" || u.Path == "" { return nil } target := u.Path if strings.HasPrefix(target, "/") { target = strings.TrimPrefix(path.Clean(target), "/") } else { target = path.Clean(path.Join(path.Dir(from), target)) } if strings.HasSuffix(u.Path, "/") { target = path.Join(target, "index.html") } if target == ".." || strings.HasPrefix(target, "../") { return fmt.Errorf("internal link escapes the site root in %s: %s", from, raw) } if _, ok := outputs[target]; ok { return nil } full := filepath.Join(s.root, filepath.FromSlash(target)) if info, statErr := os.Stat(full); statErr == nil && !info.IsDir() && !tracked[target] { return nil } indexTarget := path.Join(target, "index.html") if info, statErr := os.Stat(filepath.Join(full, "index.html")); statErr == nil && !info.IsDir() && !tracked[indexTarget] { return nil } return fmt.Errorf("broken internal link in %s: %s", from, raw) } func (s *site) readManifest() (map[string]bool, error) { tracked := map[string]bool{} raw, err := os.ReadFile(filepath.Join(s.root, manifestName)) if errors.Is(err, os.ErrNotExist) { return tracked, nil } if err != nil { return nil, err } var m manifest if err := json.Unmarshal(raw, &m); err != nil { return nil, fmt.Errorf("read %s: %w", manifestName, err) } for _, output := range m.Outputs { clean := path.Clean(output) ext := path.Ext(clean) if output == "" || output != clean || strings.Contains(output, `\`) || path.IsAbs(output) || strings.HasPrefix(clean, "../") || ext != ".html" && ext != ".xml" { return nil, fmt.Errorf("unsafe path in %s: %q", manifestName, output) } tracked[output] = true } return tracked, nil } func (s *site) write(outputs map[string][]byte, manageGitignore bool) error { tracked, err := s.readManifest() if err != nil { return err } for output := range outputs { _, statErr := os.Stat(filepath.Join(s.root, filepath.FromSlash(output))) if statErr == nil && !tracked[output] { return fmt.Errorf("refusing to overwrite untracked output %s", output) } if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { return statErr } } txn, err := os.MkdirTemp(s.root, ".weft-txn-") if err != nil { return err } defer os.RemoveAll(txn) stage := filepath.Join(txn, "stage") backup := filepath.Join(txn, "backup") if err := os.MkdirAll(stage, 0o755); err != nil { return err } for output, contents := range outputs { name := filepath.Join(stage, filepath.FromSlash(output)) if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { return err } if err := os.WriteFile(name, contents, 0o644); err != nil { return err } } paths := make([]string, 0, len(outputs)) for output := range outputs { paths = append(paths, output) } slices.Sort(paths) manifestBytes, _ := json.MarshalIndent(manifest{Outputs: paths}, "", " ") manifestBytes = append(manifestBytes, '\n') if err := os.WriteFile(filepath.Join(stage, manifestName), manifestBytes, 0o644); err != nil { return err } if manageGitignore { contents, err := s.gitignore(paths) if err != nil { return err } if err := os.WriteFile(filepath.Join(stage, ".gitignore"), contents, 0o644); err != nil { return err } } var movedOld, installed []string rollback := func() { for i := len(installed) - 1; i >= 0; i-- { _ = os.Remove(filepath.Join(s.root, filepath.FromSlash(installed[i]))) } for i := len(movedOld) - 1; i >= 0; i-- { from := filepath.Join(backup, filepath.FromSlash(movedOld[i])) to := filepath.Join(s.root, filepath.FromSlash(movedOld[i])) _ = os.MkdirAll(filepath.Dir(to), 0o755) _ = moveFile(from, to) } } moveOld := func(rel string) error { from := filepath.Join(s.root, filepath.FromSlash(rel)) if _, err := os.Stat(from); errors.Is(err, os.ErrNotExist) { return nil } else if err != nil { return err } to := filepath.Join(backup, filepath.FromSlash(rel)) if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil { return err } if err := moveFile(from, to); err != nil { return err } movedOld = append(movedOld, rel) return nil } for old := range tracked { if _, wanted := outputs[old]; !wanted { if err := moveOld(old); err != nil { rollback() return err } } } install := append(slices.Clone(paths), manifestName) if manageGitignore { install = append(install, ".gitignore") } for _, rel := range install { to := filepath.Join(s.root, filepath.FromSlash(rel)) // Leave byte-identical outputs untouched so their mtimes (and thus // Last-Modified/ETag headers) only change when content does. if existing, err := os.ReadFile(to); err == nil { staged, err := os.ReadFile(filepath.Join(stage, filepath.FromSlash(rel))) if err != nil { rollback() return err } if bytes.Equal(existing, staged) { continue } } if err := moveOld(rel); err != nil { rollback() return err } if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil { rollback() return err } if err := moveFile(filepath.Join(stage, filepath.FromSlash(rel)), to); err != nil { rollback() return err } installed = append(installed, rel) } return nil } // moveFile is os.Rename with a fallback for cross-device moves (EXDEV), which // happen when part of the site tree is a symlink onto another filesystem: the // contents are staged next to the destination and renamed into place there. func moveFile(from, to string) error { err := os.Rename(from, to) if err == nil || !errors.Is(err, syscall.EXDEV) { return err } contents, err := os.ReadFile(from) if err != nil { return err } tmp, err := os.CreateTemp(filepath.Dir(to), ".weft-move-") if err != nil { return err } _, werr := tmp.Write(contents) if cerr := tmp.Close(); werr == nil { werr = cerr } if werr == nil { werr = os.Chmod(tmp.Name(), 0o644) } if werr == nil { werr = os.Rename(tmp.Name(), to) } if werr != nil { _ = os.Remove(tmp.Name()) return werr } return os.Remove(from) } func (s *site) gitignore(outputs []string) ([]byte, error) { raw, err := os.ReadFile(filepath.Join(s.root, ".gitignore")) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, err } var kept strings.Builder inBlock, seen := false, false for _, line := range strings.SplitAfter(string(raw), "\n") { value := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") switch value { case gitignoreStart: if inBlock || seen { return nil, errors.New("invalid managed block in .gitignore") } inBlock, seen = true, true case gitignoreEnd: if !inBlock { return nil, errors.New("invalid managed block in .gitignore") } inBlock = false default: if !inBlock { kept.WriteString(line) } } } if inBlock { return nil, errors.New("unterminated managed block in .gitignore") } if kept.Len() > 0 && !strings.HasSuffix(kept.String(), "\n") { kept.WriteByte('\n') } kept.WriteString(gitignoreStart + "\n") for _, output := range outputs { if strings.ContainsAny(output, "\r\n") { return nil, fmt.Errorf("cannot add output to .gitignore: %q", output) } kept.WriteString("/" + escapeGitignore(output) + "\n") } kept.WriteString(gitignoreEnd + "\n") return []byte(kept.String()), nil } func escapeGitignore(value string) string { return strings.NewReplacer( `\`, `\\`, " ", `\ `, "#", `\#`, "!", `\!`, "[", `\[`, "]", `\]`, "*", `\*`, "?", `\?`, ).Replace(value) }