diff --git a/default.yaml b/default.yaml index 3981c82..04f5d2f 100644 --- a/default.yaml +++ b/default.yaml @@ -6,8 +6,10 @@ lint: xunitReport: "" jsonFile: "" ignoreNoqa: false - noCache: false skip: {} +cache: + directory: .mendix-cache/mxlint + enable: true modelsource: modelsource projectDirectory: . export: diff --git a/lint/cache.go b/lint/cache.go index 4d4a350..d36e7e3 100644 --- a/lint/cache.go +++ b/lint/cache.go @@ -6,10 +6,29 @@ import ( "fmt" "os" "path/filepath" + "strings" + "sync" ) const cacheVersion = "v1" +var cacheDirConfig = struct { + mu sync.RWMutex + dir string +}{} + +func SetCacheDirectory(path string) { + cacheDirConfig.mu.Lock() + defer cacheDirConfig.mu.Unlock() + cacheDirConfig.dir = strings.TrimSpace(path) +} + +func getConfiguredCacheDirectory() string { + cacheDirConfig.mu.RLock() + defer cacheDirConfig.mu.RUnlock() + return cacheDirConfig.dir +} + // CacheKey represents the unique identifier for a cached result type CacheKey struct { RuleHash string `json:"rule_hash"` @@ -25,6 +44,10 @@ type CachedTestcase struct { // getCacheDir returns the cache directory path func getCacheDir() (string, error) { + if configured := getConfiguredCacheDirectory(); configured != "" { + return configured, nil + } + homeDir, err := os.UserHomeDir() if err != nil { return "", err diff --git a/lint/config.go b/lint/config.go index 94c1480..865a96d 100644 --- a/lint/config.go +++ b/lint/config.go @@ -16,6 +16,7 @@ const configFileName = "mxlint.yaml" type Config struct { Rules ConfigRulesSpec `yaml:"rules"` Lint ConfigLintSpec `yaml:"lint"` + Cache ConfigCacheSpec `yaml:"cache"` Export ConfigExportSpec `yaml:"export"` Serve ConfigServeSpec `yaml:"serve"` Modelsource string `yaml:"modelsource"` @@ -37,10 +38,14 @@ type ConfigLintSpec struct { XunitReport string `yaml:"xunitReport"` JSONFile string `yaml:"jsonFile"` IgnoreNoqa *bool `yaml:"ignoreNoqa"` - NoCache *bool `yaml:"noCache"` Skip map[string][]ConfigSkipRule `yaml:"skip"` } +type ConfigCacheSpec struct { + Directory string `yaml:"directory"` + Enable *bool `yaml:"enable"` +} + type ConfigServeSpec struct { Port *int `yaml:"port"` Debounce *int `yaml:"debounce"` @@ -288,8 +293,11 @@ func mergeConfig(base *Config, overlay *Config) { if overlay.Lint.IgnoreNoqa != nil { base.Lint.IgnoreNoqa = overlay.Lint.IgnoreNoqa } - if overlay.Lint.NoCache != nil { - base.Lint.NoCache = overlay.Lint.NoCache + if strings.TrimSpace(overlay.Cache.Directory) != "" { + base.Cache.Directory = strings.TrimSpace(overlay.Cache.Directory) + } + if overlay.Cache.Enable != nil { + base.Cache.Enable = overlay.Cache.Enable } if overlay.Serve.Port != nil { diff --git a/main.go b/main.go index 2c17cbe..bf36e95 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "text/tabwriter" "github.com/mxlint/mxlint-cli/lint" @@ -48,6 +49,8 @@ func main() { log.SetLevel(logrus.InfoLevel) } mpr.SetLogger(log) + lint.SetConfig(config) + configureCache(config, projectDir) inputDirectory := config.ProjectDirectory outputDirectory := config.Modelsource @@ -91,6 +94,7 @@ func main() { } lint.SetLogger(log) lint.SetConfig(config) + configureCache(config, projectDir) rulesDirectory := config.Rules.Path modelDirectory := config.Modelsource @@ -113,7 +117,7 @@ func main() { config.Lint.XunitReport, config.Lint.JSONFile, boolValue(config.Lint.IgnoreNoqa, false), - !boolValue(config.Lint.NoCache, false), + boolValue(config.Cache.Enable, true), ) if err != nil { log.Errorf("lint failed: %s", err) @@ -201,6 +205,19 @@ func main() { Short: "Clear the lint results cache", Long: "Removes all cached lint results. The cache is used to speed up repeated linting operations when rules and model files haven't changed.", Run: func(cmd *cobra.Command, args []string) { + projectDir, err := os.Getwd() + if err != nil { + fmt.Printf("failed to resolve current working directory: %s\n", err) + os.Exit(1) + } + config, err := lint.LoadMergedConfigFromPath(projectDir, configPathForCommand(cmd)) + if err != nil { + fmt.Printf("failed to load configuration: %s\n", err) + os.Exit(1) + } + lint.SetConfig(config) + configureCache(config, projectDir) + log := logrus.New() if isVerbose(cmd) { log.SetLevel(logrus.DebugLevel) @@ -208,7 +225,7 @@ func main() { log.SetLevel(logrus.InfoLevel) } lint.SetLogger(log) - err := lint.ClearCache() + err = lint.ClearCache() if err != nil { log.Errorf("Failed to clear cache: %s", err) os.Exit(1) @@ -222,6 +239,19 @@ func main() { Short: "Show cache statistics", Long: "Displays information about the cached lint results, including number of entries and total size.", Run: func(cmd *cobra.Command, args []string) { + projectDir, err := os.Getwd() + if err != nil { + fmt.Printf("failed to resolve current working directory: %s\n", err) + os.Exit(1) + } + config, err := lint.LoadMergedConfigFromPath(projectDir, configPathForCommand(cmd)) + if err != nil { + fmt.Printf("failed to load configuration: %s\n", err) + os.Exit(1) + } + lint.SetConfig(config) + configureCache(config, projectDir) + log := logrus.New() if isVerbose(cmd) { log.SetLevel(logrus.DebugLevel) @@ -255,6 +285,24 @@ func main() { } } +func configureCache(config *lint.Config, projectDir string) { + if config == nil { + return + } + cacheBase := strings.TrimSpace(config.Cache.Directory) + if cacheBase == "" { + return + } + + if !filepath.IsAbs(cacheBase) { + cacheBase = filepath.Join(projectDir, cacheBase) + } + + lint.SetCacheDirectory(filepath.Join(cacheBase, "lint")) + mpr.SetPersistentYAMLCacheDirectory(filepath.Join(cacheBase, "mpr-v2-yaml")) + mpr.SetPersistentYAMLCacheEnabled(boolValue(config.Cache.Enable, true)) +} + func boolValue(value *bool, fallback bool) bool { if value == nil { return fallback diff --git a/mpr/mpr.go b/mpr/mpr.go index c95b83a..920a55e 100644 --- a/mpr/mpr.go +++ b/mpr/mpr.go @@ -10,6 +10,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "gopkg.in/yaml.v3" "go.mongodb.org/mongo-driver/bson" @@ -30,8 +31,37 @@ const ( // Per-component limit - filename or foldername can be at most 50 characters long MaxComponentLength = 50 + + // Bump this when YAML rendering semantics change. + persistentYAMLCacheVersion = "v1" ) +var persistentYAMLCacheSettings = struct { + mu sync.RWMutex + baseDir string + enabled bool +}{ + enabled: true, +} + +func SetPersistentYAMLCacheDirectory(path string) { + persistentYAMLCacheSettings.mu.Lock() + defer persistentYAMLCacheSettings.mu.Unlock() + persistentYAMLCacheSettings.baseDir = strings.TrimSpace(path) +} + +func SetPersistentYAMLCacheEnabled(enabled bool) { + persistentYAMLCacheSettings.mu.Lock() + defer persistentYAMLCacheSettings.mu.Unlock() + persistentYAMLCacheSettings.enabled = enabled +} + +func getPersistentYAMLCacheSettings() (string, bool) { + persistentYAMLCacheSettings.mu.RLock() + defer persistentYAMLCacheSettings.mu.RUnlock() + return persistentYAMLCacheSettings.baseDir, persistentYAMLCacheSettings.enabled +} + func ExportModel(inputDirectory string, outputDirectory string, raw bool, appstore bool, filter string) error { // create tmp directory in user tmp directory @@ -470,10 +500,11 @@ func getMxDocuments(units []MxUnit, folders []MxFolder) ([]MxDocument, error) { } myDocument := MxDocument{ - Name: name, - Type: unit.Contents["$Type"].(string), - Path: getMxDocumentPath(unit.ContainerID, folders), - Attributes: unit.Contents, + Name: name, + Type: unit.Contents["$Type"].(string), + Path: getMxDocumentPath(unit.ContainerID, folders), + Attributes: unit.Contents, + ContentsHash: unit.ContentsHash, } if unit.Contents["$Type"] == microflowDocumentType { @@ -557,7 +588,7 @@ func exportUnits(inputDirectory string, outputDirectory string, raw bool, filter } attributes := cleanData(document.Attributes, raw) - err = writeFile(filepath.Join(directory, adjustedFilename), attributes) + err = writeFileWithPersistentCache(filepath.Join(directory, adjustedFilename), attributes, document.ContentsHash, raw) if err != nil { log.Errorf("Error writing file: %v", err) return 0, err @@ -575,15 +606,104 @@ func exportUnits(inputDirectory string, outputDirectory string, raw bool, filter func writeFile(filepath string, contents map[string]interface{}) error { log.Debugf("Writing file %s", filepath) + yamlstring, err := renderYAML(contents) + if err != nil { + return err + } + + if err := os.WriteFile(filepath, yamlstring, 0644); err != nil { + return fmt.Errorf("error writing file: %v", err) + } + return nil +} + +func renderYAML(contents map[string]interface{}) ([]byte, error) { normalizedContents := normalizeMultilineValues(contents) yamlstring, err := yaml.Marshal(normalizedContents) if err != nil { - return fmt.Errorf("error marshaling: %v", err) + return nil, fmt.Errorf("error marshaling: %v", err) + } + return yamlstring, nil +} + +func writeFileWithPersistentCache(filepath string, contents map[string]interface{}, contentsHash string, raw bool) error { + log.Debugf("Writing file %s", filepath) + _, cacheEnabled := getPersistentYAMLCacheSettings() + if !cacheEnabled { + return writeFile(filepath, contents) + } + + // Fallback for content without a hash (e.g. MPR v1 path). + if strings.TrimSpace(contentsHash) == "" { + return writeFile(filepath, contents) + } + + cachedYAML, found, err := readYAMLFromPersistentCache(contentsHash, raw) + if err != nil { + return err + } + if found { + if err := os.WriteFile(filepath, cachedYAML, 0644); err != nil { + return fmt.Errorf("error writing cached file: %v", err) + } + return nil + } + + yamlstring, err := renderYAML(contents) + if err != nil { + return err } if err := os.WriteFile(filepath, yamlstring, 0644); err != nil { return fmt.Errorf("error writing file: %v", err) } + + if err := writeYAMLToPersistentCache(contentsHash, raw, yamlstring); err != nil { + log.Debugf("Could not persist YAML cache for hash %s: %v", contentsHash, err) + } + + return nil +} + +func getPersistentYAMLCacheDir() string { + configured, _ := getPersistentYAMLCacheSettings() + return strings.TrimSpace(configured) +} + +func getPersistentYAMLCachePath(contentsHash string, raw bool) string { + cacheKey := fmt.Sprintf("%s|raw=%t|%s", persistentYAMLCacheVersion, raw, contentsHash) + sum := sha256.Sum256([]byte(cacheKey)) + return filepath.Join(getPersistentYAMLCacheDir(), hex.EncodeToString(sum[:])+".yaml") +} + +func readYAMLFromPersistentCache(contentsHash string, raw bool) ([]byte, bool, error) { + cacheDir := getPersistentYAMLCacheDir() + if cacheDir == "" { + return nil, false, nil + } + cachePath := getPersistentYAMLCachePath(contentsHash, raw) + data, err := os.ReadFile(cachePath) + if err != nil { + if os.IsNotExist(err) { + return nil, false, nil + } + return nil, false, fmt.Errorf("error reading persistent YAML cache: %v", err) + } + return data, true, nil +} + +func writeYAMLToPersistentCache(contentsHash string, raw bool, data []byte) error { + cacheDir := getPersistentYAMLCacheDir() + if cacheDir == "" { + return nil + } + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return fmt.Errorf("error creating persistent YAML cache directory: %v", err) + } + cachePath := getPersistentYAMLCachePath(contentsHash, raw) + if err := os.WriteFile(cachePath, data, 0644); err != nil { + return fmt.Errorf("error writing persistent YAML cache file: %v", err) + } return nil } @@ -810,6 +930,15 @@ func syncDirectories(src, dst string) error { // copyFile copies a single file from src to dst func copyFile(src, dst string, mode os.FileMode) error { + // Skip rewriting unchanged files to speed up repeated exports. + same, err := filesHaveSameContent(src, dst) + if err != nil { + return fmt.Errorf("failed to compare files: %v", err) + } + if same { + return nil + } + // Open the source file srcFile, err := os.Open(src) if err != nil { @@ -832,6 +961,49 @@ func copyFile(src, dst string, mode os.FileMode) error { return nil } +func filesHaveSameContent(src, dst string) (bool, error) { + srcInfo, err := os.Stat(src) + if err != nil { + return false, err + } + dstInfo, err := os.Stat(dst) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + // Fast path: different size means definitely changed. + if srcInfo.Size() != dstInfo.Size() { + return false, nil + } + + srcHash, err := hashFile(src) + if err != nil { + return false, err + } + dstHash, err := hashFile(dst) + if err != nil { + return false, err + } + return srcHash == dstHash, nil +} + +func hashFile(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + // FileNode represents a file or directory in the file structure type FileNode struct { Name string `yaml:"name"` diff --git a/mpr/mpr_test.go b/mpr/mpr_test.go index 083ab5e..bc3dd20 100644 --- a/mpr/mpr_test.go +++ b/mpr/mpr_test.go @@ -1,10 +1,12 @@ package mpr import ( + "bytes" "os" "path/filepath" "strings" "testing" + "time" "gopkg.in/yaml.v3" "go.mongodb.org/mongo-driver/bson" @@ -572,6 +574,57 @@ func TestWriteFile(t *testing.T) { } } +func TestWriteFileWithPersistentCache(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "mpr-cache-test-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + cacheDir := filepath.Join(tmpDir, "cache") + SetPersistentYAMLCacheDirectory(cacheDir) + SetPersistentYAMLCacheEnabled(true) + t.Cleanup(func() { + SetPersistentYAMLCacheDirectory("") + SetPersistentYAMLCacheEnabled(true) + }) + + contents := map[string]interface{}{ + "Name": "TestDocument", + "Value": "if $x = 1 then true\nelse false", + } + hash := "test-hash-123" + + firstOutput := filepath.Join(tmpDir, "first.yaml") + if err := writeFileWithPersistentCache(firstOutput, contents, hash, false); err != nil { + t.Fatalf("writeFileWithPersistentCache() first write failed: %v", err) + } + + cachePath := getPersistentYAMLCachePath(hash, false) + if _, err := os.Stat(cachePath); err != nil { + t.Fatalf("Expected cache file to exist, got error: %v", err) + } + + // Overwrite cache file with sentinel content to verify second write reads cache. + sentinel := []byte("Cached: true\n") + if err := os.WriteFile(cachePath, sentinel, 0644); err != nil { + t.Fatalf("Failed to overwrite cache file: %v", err) + } + + secondOutput := filepath.Join(tmpDir, "second.yaml") + if err := writeFileWithPersistentCache(secondOutput, contents, hash, false); err != nil { + t.Fatalf("writeFileWithPersistentCache() second write failed: %v", err) + } + + written, err := os.ReadFile(secondOutput) + if err != nil { + t.Fatalf("Failed to read second output file: %v", err) + } + if !bytes.Equal(written, sentinel) { + t.Fatalf("Expected second write to use cached YAML content, got: %s", string(written)) + } +} + func TestSyncDirectories(t *testing.T) { // Create temporary directories for testing srcDir, err := os.MkdirTemp("", "mpr-test-src-*") @@ -688,6 +741,40 @@ func TestCopyFile(t *testing.T) { } }) } + + t.Run("skip unchanged destination content", func(t *testing.T) { + src := filepath.Join(tmpDir, "unchanged-source.txt") + dst := filepath.Join(tmpDir, "unchanged-destination.txt") + content := []byte("same content") + + if err := os.WriteFile(src, content, 0644); err != nil { + t.Fatalf("Failed to write source file: %v", err) + } + if err := os.WriteFile(dst, content, 0644); err != nil { + t.Fatalf("Failed to write destination file: %v", err) + } + + beforeInfo, err := os.Stat(dst) + if err != nil { + t.Fatalf("Failed to stat destination file before copy: %v", err) + } + + // Ensure modtime resolution difference is observable on most filesystems. + time.Sleep(20 * time.Millisecond) + + if err := copyFile(src, dst, 0644); err != nil { + t.Fatalf("copyFile() unexpected error: %v", err) + } + + afterInfo, err := os.Stat(dst) + if err != nil { + t.Fatalf("Failed to stat destination file after copy: %v", err) + } + + if !afterInfo.ModTime().Equal(beforeInfo.ModTime()) { + t.Errorf("Expected unchanged file to be skipped, but modtime changed from %v to %v", beforeInfo.ModTime(), afterInfo.ModTime()) + } + }) } func TestGetMxDocuments(t *testing.T) { diff --git a/mpr/mx_units_v2.go b/mpr/mx_units_v2.go index 1f39f2d..39dd8d7 100644 --- a/mpr/mx_units_v2.go +++ b/mpr/mx_units_v2.go @@ -1,8 +1,10 @@ package mpr import ( + "crypto/sha256" "database/sql" "encoding/base64" + "encoding/hex" "fmt" "os" "path/filepath" @@ -50,6 +52,8 @@ func readMxUnitsV2(inputDirectory string) ([]MxUnit, error) { if err != nil { return fmt.Errorf("error reading file %s: %v", path, err) } + contentHash := sha256.Sum256(contents) + contentHashHex := hex.EncodeToString(contentHash[:]) var result bson.M if err := bson.Unmarshal(contents, &result); err != nil { @@ -144,6 +148,7 @@ func readMxUnitsV2(inputDirectory string) ([]MxUnit, error) { } units[idx].Contents = result + units[idx].ContentsHash = contentHashHex } return nil }) diff --git a/mpr/types.go b/mpr/types.go index 730c4d7..01deede 100644 --- a/mpr/types.go +++ b/mpr/types.go @@ -11,6 +11,7 @@ type MxUnit struct { ContainerID string `yaml:"ContainerID"` ContainmentName string `yaml:"ContainmentName"` Contents map[string]interface{} `yaml:"Contents"` + ContentsHash string `yaml:"ContentsHash,omitempty"` } type MxDocument struct { @@ -18,6 +19,7 @@ type MxDocument struct { Type string `yaml:"Type"` Path string `yaml:"Path"` Attributes map[string]interface{} `yaml:"Attributes"` + ContentsHash string `yaml:"ContentsHash,omitempty"` } type MxModule struct { diff --git a/serve/serve.go b/serve/serve.go index 22edffa..5136e6c 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -51,6 +51,7 @@ func runServe(cmd *cobra.Command, args []string) { os.Exit(1) } lint.SetConfig(config) + configureCacheForServe(config, projectDir) inputDirectory := config.ProjectDirectory outputDirectory := config.Modelsource @@ -222,7 +223,7 @@ func runServe(cmd *cobra.Command, args []string) { lintErr = fmt.Errorf("lint operation panicked: %v", r) } }() - results, lintErr = lint.EvalAllWithResults(rulesDirectory, outputDirectory, "", "", false, true) + results, lintErr = lint.EvalAllWithResults(rulesDirectory, outputDirectory, "", "", false, boolValue(config.Cache.Enable, true)) }() if lintErr != nil { @@ -322,6 +323,30 @@ func intValue(value *int, fallback int) int { return *value } +func boolValue(value *bool, fallback bool) bool { + if value == nil { + return fallback + } + return *value +} + +func configureCacheForServe(config *lint.Config, projectDir string) { + if config == nil { + return + } + cacheBase := strings.TrimSpace(config.Cache.Directory) + if cacheBase == "" { + return + } + if !filepath.IsAbs(cacheBase) { + cacheBase = filepath.Join(projectDir, cacheBase) + } + + lint.SetCacheDirectory(filepath.Join(cacheBase, "lint")) + mpr.SetPersistentYAMLCacheDirectory(filepath.Join(cacheBase, "mpr-v2-yaml")) + mpr.SetPersistentYAMLCacheEnabled(boolValue(config.Cache.Enable, true)) +} + // addDirsRecursive adds all directories recursively to the watcher except the output directory func addDirsRecursive(watcher *fsnotify.Watcher, root string, excludeDir string, log *logrus.Logger) error { excludePath, err := filepath.Abs(excludeDir)