Implement on-demand source processing

README.md

@@ -64,8 +64,6 @@ The available arguments are:
   * `-code-path`: (Optional) Path to the source code root directory.
   * `-docs-path`: (Optional) Path to the documentation root directory.
   * `-config-path`: (Optional) Path to a YAML configuration file containing `code-path` and `docs-path`.
-  * `-code-includes`: (Optional) Comma-separated glob patterns for source files to include (e.g., `"**/*.java,**/*.gradle"`). Defaults to `"**/*.*"`.
-  * `-code-excludes`: (Optional) Comma-separated glob patterns for source files to exclude.
   * `-doc-includes`: (Optional) Comma-separated glob patterns for documentation files to include. Defaults to `"**/*.md,**/*.html"`.
   * `-fragments-path`: (Optional) Directory for storing code fragments. Defaults to `./build/fragments`.
   * `-separator`: (Optional) String used to separate joined code fragments. Defaults to `...`.
@@ -87,7 +85,6 @@ Optional settings can be defined in a YAML configuration file:
 ```yaml
 code-path: path/to/code/root
 docs-path: path/to/docs/root
-code-includes: "**/*.java,**/*.gradle"
 doc-excludes: "**/*-old.*,**/deprecated/*.*"
 ```
 
@@ -98,7 +95,6 @@ embeddings:
   - name: java
     code-path: path/to/code/root/java
     docs-path: path/to/java/docs
-    code-includes: "**/*.java"
   - name: kotlin
     code-path:
       - name: samples
@@ -138,8 +134,6 @@ The available fields for the configuration file are:
       but this may lead to fragments being overwritten if they have the same relative path and name.
 
   * `docs-path`: (Mandatory) Path to the documentation root.
-  * `code-includes`: (Optional) Glob patterns for source files to include.
-    It may be represented as a comma-separated string list or as a YAML sequence.
   * `doc-excludes`: (Optional) Glob patterns for documentation files to exclude.
     It may be represented as a comma-separated string list or as a YAML sequence.
   * `doc-includes`: (Optional) Glob patterns for documentation files to include.

cli/cli.go

@@ -28,7 +28,6 @@ import (
 	"embed-code/embed-code-go/analyzing"
 	"embed-code/embed-code-go/configuration"
 	"embed-code/embed-code-go/embedding"
-	"embed-code/embed-code-go/fragmentation"
 
 	"gopkg.in/yaml.v3"
 )
@@ -39,12 +38,6 @@ import (
 //
 // BaseDocsPath — a path to a root directory with docs files.
 //
-// CodeIncludes — a StringList with patterns for filtering the code files
-// to be considered.
-// Directories are never matched by these patterns.
-// For example, "**/*.java,**/*.gradle".
-// The default value is "**/*.*".
-//
 // DocIncludes — a StringList with patterns for filtering files
 // in which we should look for embedding instructions.
 // The patterns are resolved relatively to the `documentation_root`.
@@ -73,7 +66,6 @@ import (
 type Config struct {
 	BaseCodePaths _type.NamedPathList `yaml:"code-path"`
 	BaseDocsPath  string              `yaml:"docs-path"`
-	CodeIncludes  _type.StringList    `yaml:"code-includes"`
 	DocIncludes   _type.StringList    `yaml:"doc-includes"`
 	DocExcludes   _type.StringList    `yaml:"doc-excludes"`
 	FragmentsPath string              `yaml:"fragments-path"`
@@ -90,7 +82,6 @@ type EmbeddingConfig struct {
 	Name          string              `yaml:"name"`
 	CodePaths     _type.NamedPathList `yaml:"code-path"`
 	DocsPath      string              `yaml:"docs-path"`
-	CodeIncludes  _type.StringList    `yaml:"code-includes"`
 	DocIncludes   _type.StringList    `yaml:"doc-includes"`
 	DocExcludes   _type.StringList    `yaml:"doc-excludes"`
 	FragmentsPath string              `yaml:"fragments-path"`
@@ -99,11 +90,8 @@ type EmbeddingConfig struct {
 
 // EmbedCodeSamplesResult is result of the EmbedCodeSamples method.
 //
-// WriteFragmentFilesResult the result of code fragmentation.
-//
 // EmbedAllResult the result of embedding code fragments in the documentation.
 type EmbedCodeSamplesResult struct {
-	fragmentation.WriteFragmentFilesResult
 	embedding.EmbedAllResult
 }
 
@@ -118,19 +106,16 @@ const (
 //
 // config — a configuration for checking code samples.
 func CheckCodeSamples(config configuration.Configuration) {
-	fragmentation.WriteFragmentFiles(config)
 	embedding.CheckUpToDate(config)
 }
 
 // EmbedCodeSamples embeds code fragments in documentation files.
 //
 // config — a configuration for embedding.
 func EmbedCodeSamples(config configuration.Configuration) EmbedCodeSamplesResult {
-	fragmentationResult := fragmentation.WriteFragmentFiles(config)
 	embeddingResult := embedding.EmbedAll(config)
 	embedding.CheckUpToDate(config)
 	return EmbedCodeSamplesResult{
-		fragmentationResult,
 		embeddingResult,
 	}
 }
@@ -139,9 +124,7 @@ func EmbedCodeSamples(config configuration.Configuration) EmbedCodeSamplesResult
 //
 // config — a configuration for embedding.
 func AnalyzeCodeSamples(config configuration.Configuration) {
-	fragmentation.WriteFragmentFiles(config)
 	analyzing.AnalyzeAll(config)
-	fragmentation.CleanFragmentFiles(config)
 }
 
 // ReadArgs reads user-specified args from the command line.
@@ -150,8 +133,6 @@ func AnalyzeCodeSamples(config configuration.Configuration) {
 func ReadArgs() Config {
 	codePath := flag.String("code-path", "", "a path to a root directory with code files")
 	docsPath := flag.String("docs-path", "", "a path to a root directory with docs files")
-	codeIncludes := flag.String("code-includes", "",
-		"a comma-separated string of glob patterns for code files to include")
 	docIncludes := flag.String("doc-includes", "",
 		"a comma-separated string of glob patterns for docs files to include")
 	docExcludes := flag.String("doc-excludes", "",
@@ -173,7 +154,6 @@ func ReadArgs() Config {
 	return Config{
 		BaseCodePaths: _type.NamedPathList{_type.NamedPath{Path: *codePath}},
 		BaseDocsPath:  *docsPath,
-		CodeIncludes:  parseListArgument(*codeIncludes),
 		DocIncludes:   parseListArgument(*docIncludes),
 		DocExcludes:   parseListArgument(*docExcludes),
 		FragmentsPath: *fragmentsPath,
@@ -195,9 +175,6 @@ func FillArgsFromConfigFile(args Config) (Config, error) {
 	args.BaseDocsPath = configFields.BaseDocsPath
 	args.BaseCodePaths = configFields.BaseCodePaths
 
-	if len(configFields.CodeIncludes) > 0 {
-		args.CodeIncludes = configFields.CodeIncludes
-	}
 	if len(configFields.Embeddings) > 0 {
 		args.Embeddings = configFields.Embeddings
 	}
@@ -248,9 +225,6 @@ func configFromEmbedding(embedding EmbeddingConfig) configuration.Configuration
 	embedCodeConfig.CodeRoots = embedding.CodePaths
 	embedCodeConfig.DocumentationRoot = embedding.DocsPath
 
-	if len(embedding.CodeIncludes) > 0 {
-		embedCodeConfig.CodeIncludes = embedding.CodeIncludes
-	}
 	if len(embedding.DocIncludes) > 0 {
 		embedCodeConfig.DocIncludes = embedding.DocIncludes
 	}
@@ -272,9 +246,6 @@ func configFromEmbedding(embedding EmbeddingConfig) configuration.Configuration
 func configWithOptionalParams(userArgs Config) configuration.Configuration {
 	embedCodeConfig := configuration.NewConfiguration()
 
-	if len(userArgs.CodeIncludes) > 0 {
-		embedCodeConfig.CodeIncludes = userArgs.CodeIncludes
-	}
 	if len(userArgs.DocIncludes) > 0 {
 		embedCodeConfig.DocIncludes = userArgs.DocIncludes
 	}

cli/cli_test.go

@@ -155,9 +155,9 @@ var _ = Describe("CLI validation", func() {
 
 		It("should fail validation when embeddings and root optional params are set at the same time", func() {
 			invalidConfig := cli.Config{
-				Mode:         cli.ModeCheck,
-				CodeIncludes: []string{"**/*.java"},
-				Embeddings:   []cli.EmbeddingConfig{baseEmbeddingConfig()},
+				Mode:        cli.ModeCheck,
+				DocIncludes: []string{"**/*.md"},
+				Embeddings:  []cli.EmbeddingConfig{baseEmbeddingConfig()},
 			}
 
 			Expect(cli.ValidateConfig(invalidConfig)).Error().Should(HaveOccurred())

cli/cli_validation.go

@@ -256,14 +256,12 @@ func verifyDuplicateEmbeddingDocsPaths(embeddings []EmbeddingConfig) {
 
 // validateOptionalParamsSet reports whether at least one optional config is set.
 func validateOptionalParamsSet(config Config) bool {
-	isCodeIncludesSet := len(config.CodeIncludes) > 0
 	isDocIncludesSet := len(config.DocIncludes) > 0
 	isDocExcludesSet := len(config.DocExcludes) > 0
 	isSeparatorSet := isNotEmpty(config.Separator)
 	isFragmentPathSet := isNotEmpty(config.FragmentsPath)
 
-	return isCodeIncludesSet || isDocIncludesSet || isFragmentPathSet ||
-		isSeparatorSet || isDocExcludesSet
+	return isDocIncludesSet || isFragmentPathSet || isSeparatorSet || isDocExcludesSet
 }
 
 // validatePathSet reports whether path is set and checks if it exists.

configuration/configuration.go

@@ -28,7 +28,6 @@ const (
 	DefaultFragmentsDir = "./build/fragments"
 )
 
-var DefaultInclude = []string{"**/*.*"}
 var DefaultDocIncludes = []string{"**/*.md", "**/*.html"}
 
 // Configuration contains the settings for the plugin to work.
@@ -53,15 +52,6 @@ type Configuration struct {
 	// DocumentationRoot is a root directory of the documentation files.
 	DocumentationRoot string
 
-	// CodeIncludes is a list of patterns for filtering the code files to be considered.
-	//
-	// Directories are never matched by these patterns.
-	//
-	// For example, ["**/*.java", "**/*.gradle"].
-	//
-	// The default value is "**/*".
-	CodeIncludes []string
-
 	// DocIncludes is a list of patterns for filtering files in which we should look for embedding
 	// instructions.
 	//
@@ -101,7 +91,6 @@ type Configuration struct {
 // NewConfiguration builds the default config.
 func NewConfiguration() Configuration {
 	return Configuration{
-		CodeIncludes: DefaultInclude,
 		DocIncludes:  DefaultDocIncludes,
 		FragmentsDir: DefaultFragmentsDir,
 		Separator:    DefaultSeparator,

embedding/embedding_test.go

@@ -107,6 +107,23 @@ var _ = Describe("Embedding", func() {
 		Expect(processor.IsUpToDate()).Should(BeTrue())
 	})
 
+	It("should embed directly from source without writing fragment files", func() {
+		config.CodeRoots = _type.NamedPathList{_type.NamedPath{Path: "../test/resources/code/java"}}
+		config.FragmentsDir = "../test/lazy-fragments"
+		if err := os.RemoveAll(config.FragmentsDir); err != nil {
+			Fail(err.Error())
+		}
+		docPath := fmt.Sprintf("%s/doc.md", config.DocumentationRoot)
+		processor := embedding.NewProcessor(docPath, config)
+
+		Expect(processor.Embed()).Error().ShouldNot(HaveOccurred())
+		exists, err := files.IsDirExist(config.FragmentsDir)
+
+		Expect(err).ShouldNot(HaveOccurred())
+		Expect(exists).Should(BeFalse())
+		Expect(processor.IsUpToDate()).Should(BeTrue())
+	})
+
 	It("should embed with multi lined tag attributes", func() {
 		docPath := fmt.Sprintf("%s/multi-lined-valid-tag-attributes.md", config.DocumentationRoot)
 		processor := embedding.NewProcessor(docPath, config)
@@ -160,7 +177,7 @@ var _ = Describe("Embedding", func() {
 		Expect(processor.IsUpToDate()).Should(BeTrue())
 	})
 
-	It("should not embed to a file matched the `code-excludes` pattern", func() {
+	It("should not embed to a file matched the `doc-excludes` pattern", func() {
 		config.DocExcludes = []string{"**/excluded-doc.*"}
 
 		docPath := fmt.Sprintf("%s/excluded-doc.md", config.DocumentationRoot)
@@ -174,8 +191,7 @@ var _ = Describe("Embedding", func() {
 func buildConfigWithPreparedFragments() configuration.Configuration {
 	var config = configuration.NewConfiguration()
 	config.DocumentationRoot = temporaryTestDir
-	config.CodeRoots = _type.NamedPathList{_type.NamedPath{Path: "../test/resources/code"}}
-	config.FragmentsDir = "../test/resources/prepared-fragments"
+	config.CodeRoots = _type.NamedPathList{_type.NamedPath{Path: "../test/resources/code/java"}}
 
 	return config
 }

embedding/parsing/instruction.go

@@ -100,25 +100,15 @@ func NewInstruction(
 //
 // Returns an error if there was an error during reading the content.
 func (e Instruction) Content() ([]string, error) {
-	fragmentName := e.Fragment
-	if fragmentName == "" {
-		fragmentName = fragmentation.DefaultFragmentName
-	}
-	file := fragmentation.FragmentFile{
-		CodePath:      e.CodeFile,
-		FragmentName:  fragmentName,
-		Configuration: e.Configuration,
+	fileContent, err := fragmentation.ResolveContent(e.CodeFile, e.Fragment, e.Configuration)
+	if err != nil {
+		return nil, err
 	}
 	if e.StartPattern != nil || e.EndPattern != nil {
-		fileContent, err := file.Content()
-		if err != nil {
-			return nil, err
-		}
-
 		return e.matchingLines(fileContent), nil
 	}
 
-	return file.Content()
+	return fileContent, nil
 }
 
 // Returns string representation of Instruction.

embedding/parsing/instruction_test.go

@@ -301,8 +301,7 @@ func getXMLExtractionContent(fileName string, params TestInstructionParams,
 func buildConfigWithPreparedFragments() configuration.Configuration {
 	var config = configuration.NewConfiguration()
 	config.DocumentationRoot = "../../test/resources/docs"
-	config.CodeRoots = _type.NamedPathList{_type.NamedPath{Path: "../../test/resources/code"}}
-	config.FragmentsDir = "../../test/resources/prepared-fragments"
+	config.CodeRoots = _type.NamedPathList{_type.NamedPath{Path: "../../test/resources/code/java"}}
 
 	return config
 }

fragmentation/fragment.go

@@ -44,20 +44,6 @@ func CreateDefaultFragment() Fragment {
 	}
 }
 
-// WriteTo takes given lines, unites them into a text and writes it into given file.
-//
-// file — a FragmentFile to write the lines to.
-//
-// lines — a list of strings to write.
-//
-// separator — string to insert between multiple partitions of a single fragment.
-//
-// Creates the file if not exists and overwrites if exists.
-func (f Fragment) WriteTo(file FragmentFile, lines []string, separator string) {
-	text := f.text(lines, separator)
-	file.Write(text)
-}
-
 func (f Fragment) isDefault() bool {
 	return f.Name == DefaultFragmentName
 }