diff --git a/CHANGELOG.md b/CHANGELOG.md index a3a20f55e..42b5c1fd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Added environment variable support for repo sync interval settings (`REINDEX_INTERVAL_MS`, `RESYNC_CONNECTION_INTERVAL_MS`, `REINDEX_REPO_POLLING_INTERVAL_MS`, `RESYNC_CONNECTION_POLLING_INTERVAL_MS`). [#1057](https://github.com/sourcebot-dev/sourcebot/pull/1057) + ## [4.16.3] - 2026-03-27 ### Added diff --git a/docs/docs/configuration/config-file.mdx b/docs/docs/configuration/config-file.mdx index 4e63cde85..178b57294 100644 --- a/docs/docs/configuration/config-file.mdx +++ b/docs/docs/configuration/config-file.mdx @@ -40,10 +40,10 @@ The following are settings that can be provided in your config file to modify So |-------------------------------------------------|---------|------------|---------|----------------------------------------------------------------------------------------| | `maxFileSize` | number | 2 MB | 1 | Maximum size (bytes) of a file to index. Files exceeding this are skipped. | | `maxTrigramCount` | number | 20 000 | 1 | Maximum trigrams per document. Larger files are skipped. | -| `reindexIntervalMs` | number | 1 hour | 1 | Interval at which all repositories are re‑indexed. | -| `resyncConnectionIntervalMs` | number | 24 hours | 1 | Interval for checking connections that need re‑syncing. | -| `resyncConnectionPollingIntervalMs` | number | 1 second | 1 | DB polling rate for connections that need re‑syncing. | -| `reindexRepoPollingIntervalMs` | number | 1 second | 1 | DB polling rate for repos that should be re‑indexed. | +| `reindexIntervalMs` | number | 1 hour | 1 | Interval at which all repositories are re‑indexed. Can be overridden with `REINDEX_INTERVAL_MS` env var. | +| `resyncConnectionIntervalMs` | number | 24 hours | 1 | Interval for checking connections that need re‑syncing. Can be overridden with `RESYNC_CONNECTION_INTERVAL_MS` env var. | +| `resyncConnectionPollingIntervalMs` | number | 1 second | 1 | DB polling rate for connections that need re‑syncing. Can be overridden with `RESYNC_CONNECTION_POLLING_INTERVAL_MS` env var. | +| `reindexRepoPollingIntervalMs` | number | 1 second | 1 | DB polling rate for repos that should be re‑indexed. Can be overridden with `REINDEX_REPO_POLLING_INTERVAL_MS` env var. | | `maxConnectionSyncJobConcurrency` | number | 8 | 1 | Concurrent connection‑sync jobs. | | `maxRepoIndexingJobConcurrency` | number | 8 | 1 | Concurrent repo‑indexing jobs. | | `maxRepoGarbageCollectionJobConcurrency` | number | 8 | 1 | Concurrent repo‑garbage‑collection jobs. | diff --git a/docs/docs/configuration/environment-variables.mdx b/docs/docs/configuration/environment-variables.mdx index 33a1bff2e..90eb546f5 100644 --- a/docs/docs/configuration/environment-variables.mdx +++ b/docs/docs/configuration/environment-variables.mdx @@ -28,6 +28,10 @@ The following environment variables allow you to configure your Sourcebot deploy | `REDIS_REMOVE_ON_FAIL` | `100` |

Controls how many failed jobs are allowed to remain in Redis queues

| | `REPO_SYNC_RETRY_BASE_SLEEP_SECONDS` | `60` |

The base sleep duration (in seconds) for exponential backoff when retrying repository sync operations that fail

| | `GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS` | `600` |

The timeout duration (in seconds) for GitLab client queries

| +| `REINDEX_INTERVAL_MS` | `3600000` (1 hour) |

The interval (in milliseconds) at which all repositories are re-indexed. Overrides `settings.reindexIntervalMs` from the config file.

| +| `RESYNC_CONNECTION_INTERVAL_MS` | `86400000` (24 hours) |

The interval (in milliseconds) for checking connections that need re-syncing. Overrides `settings.resyncConnectionIntervalMs` from the config file.

| +| `REINDEX_REPO_POLLING_INTERVAL_MS` | `1000` (1 second) |

The polling rate (in milliseconds) at which the database should be checked for repos that need re-indexing. Overrides `settings.reindexRepoPollingIntervalMs` from the config file.

| +| `RESYNC_CONNECTION_POLLING_INTERVAL_MS` | `1000` (1 second) |

The polling rate (in milliseconds) at which the database should be checked for connections that need re-syncing. Overrides `settings.resyncConnectionPollingIntervalMs` from the config file.

| | `SMTP_CONNECTION_URL` | `-` |

The url to the SMTP service used for sending transactional emails. See [this doc](/docs/configuration/transactional-emails) for more info.

You can also use `SMTP_HOST`, `SMTP_PORT`, `SMTP_USERNAME`, and `SMTP_PASSWORD` to construct the SMTP connection url.

| | `SMTP_HOST` | `-` |

The hostname of the SMTP server. Used to construct `SMTP_CONNECTION_URL` when individual SMTP variables are provided.

| | `SMTP_PORT` | `-` |

The port of the SMTP server.

| diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 95006d875..2baa65a46 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -296,6 +296,12 @@ const options = { CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), + // Repo sync interval settings (override config file settings) + REINDEX_INTERVAL_MS: z.coerce.number().int().positive().optional(), + RESYNC_CONNECTION_INTERVAL_MS: z.coerce.number().int().positive().optional(), + REINDEX_REPO_POLLING_INTERVAL_MS: z.coerce.number().int().positive().optional(), + RESYNC_CONNECTION_POLLING_INTERVAL_MS: z.coerce.number().int().positive().optional(), + GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10), SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"), diff --git a/packages/shared/src/utils.test.ts b/packages/shared/src/utils.test.ts index c346c2c5a..b10729d15 100644 --- a/packages/shared/src/utils.test.ts +++ b/packages/shared/src/utils.test.ts @@ -9,15 +9,42 @@ vi.mock('fs/promises', () => ({ readFile: vi.fn().mockResolvedValue('{}'), })); +// Mock env module to allow overriding env vars in tests +vi.mock('./env.server.js', async (importOriginal) => { + const original = await importOriginal() as Record; + return { + ...original, + env: { + ...(original.env as Record), + REINDEX_INTERVAL_MS: undefined, + RESYNC_CONNECTION_INTERVAL_MS: undefined, + REINDEX_REPO_POLLING_INTERVAL_MS: undefined, + RESYNC_CONNECTION_POLLING_INTERVAL_MS: undefined, + }, + }; +}); + const mockConfigFile = (settings?: object) => { vi.mocked(readFile).mockResolvedValueOnce( JSON.stringify(settings !== undefined ? { settings } : {}) as any ); }; +const mockEnvVars = async (envOverrides: Record) => { + const envModule = await import('./env.server.js'); + Object.assign(envModule.env, envOverrides); +}; + describe('getConfigSettings', () => { - beforeEach(() => { + beforeEach(async () => { vi.mocked(readFile).mockResolvedValue('{}' as any); + // Reset env vars to undefined before each test + await mockEnvVars({ + REINDEX_INTERVAL_MS: undefined, + RESYNC_CONNECTION_INTERVAL_MS: undefined, + REINDEX_REPO_POLLING_INTERVAL_MS: undefined, + RESYNC_CONNECTION_POLLING_INTERVAL_MS: undefined, + }); }); test('returns DEFAULT_CONFIG_SETTINGS when no config path is provided', async () => { @@ -94,4 +121,97 @@ describe('getConfigSettings', () => { ); }); }); + + describe('env var overrides for sync intervals', () => { + test('REINDEX_INTERVAL_MS env var overrides config file setting', async () => { + await mockEnvVars({ REINDEX_INTERVAL_MS: 7200000 }); + mockConfigFile({ reindexIntervalMs: 1800000 }); + const result = await getConfigSettings('/config.json'); + expect(result.reindexIntervalMs).toBe(7200000); + }); + + test('REINDEX_INTERVAL_MS env var overrides default when no config', async () => { + await mockEnvVars({ REINDEX_INTERVAL_MS: 7200000 }); + const result = await getConfigSettings(undefined); + expect(result.reindexIntervalMs).toBe(7200000); + }); + + test('RESYNC_CONNECTION_INTERVAL_MS env var overrides config file setting', async () => { + await mockEnvVars({ RESYNC_CONNECTION_INTERVAL_MS: 43200000 }); + mockConfigFile({ resyncConnectionIntervalMs: 86400000 }); + const result = await getConfigSettings('/config.json'); + expect(result.resyncConnectionIntervalMs).toBe(43200000); + }); + + test('RESYNC_CONNECTION_INTERVAL_MS env var overrides default when no config', async () => { + await mockEnvVars({ RESYNC_CONNECTION_INTERVAL_MS: 43200000 }); + const result = await getConfigSettings(undefined); + expect(result.resyncConnectionIntervalMs).toBe(43200000); + }); + + test('REINDEX_REPO_POLLING_INTERVAL_MS env var overrides config file setting', async () => { + await mockEnvVars({ REINDEX_REPO_POLLING_INTERVAL_MS: 5000 }); + mockConfigFile({ reindexRepoPollingIntervalMs: 2000 }); + const result = await getConfigSettings('/config.json'); + expect(result.reindexRepoPollingIntervalMs).toBe(5000); + }); + + test('REINDEX_REPO_POLLING_INTERVAL_MS env var overrides default when no config', async () => { + await mockEnvVars({ REINDEX_REPO_POLLING_INTERVAL_MS: 5000 }); + const result = await getConfigSettings(undefined); + expect(result.reindexRepoPollingIntervalMs).toBe(5000); + }); + + test('RESYNC_CONNECTION_POLLING_INTERVAL_MS env var overrides config file setting', async () => { + await mockEnvVars({ RESYNC_CONNECTION_POLLING_INTERVAL_MS: 3000 }); + mockConfigFile({ resyncConnectionPollingIntervalMs: 1000 }); + const result = await getConfigSettings('/config.json'); + expect(result.resyncConnectionPollingIntervalMs).toBe(3000); + }); + + test('RESYNC_CONNECTION_POLLING_INTERVAL_MS env var overrides default when no config', async () => { + await mockEnvVars({ RESYNC_CONNECTION_POLLING_INTERVAL_MS: 3000 }); + const result = await getConfigSettings(undefined); + expect(result.resyncConnectionPollingIntervalMs).toBe(3000); + }); + + test('multiple env vars can be set simultaneously', async () => { + await mockEnvVars({ + REINDEX_INTERVAL_MS: 7200000, + RESYNC_CONNECTION_INTERVAL_MS: 43200000, + REINDEX_REPO_POLLING_INTERVAL_MS: 5000, + RESYNC_CONNECTION_POLLING_INTERVAL_MS: 3000, + }); + mockConfigFile({ + reindexIntervalMs: 1800000, + resyncConnectionIntervalMs: 86400000, + reindexRepoPollingIntervalMs: 2000, + resyncConnectionPollingIntervalMs: 1000, + }); + const result = await getConfigSettings('/config.json'); + expect(result.reindexIntervalMs).toBe(7200000); + expect(result.resyncConnectionIntervalMs).toBe(43200000); + expect(result.reindexRepoPollingIntervalMs).toBe(5000); + expect(result.resyncConnectionPollingIntervalMs).toBe(3000); + }); + + test('config file values are used when env vars are not set', async () => { + mockConfigFile({ + reindexIntervalMs: 1800000, + resyncConnectionIntervalMs: 43200000, + }); + const result = await getConfigSettings('/config.json'); + expect(result.reindexIntervalMs).toBe(1800000); + expect(result.resyncConnectionIntervalMs).toBe(43200000); + }); + + test('defaults are used when neither env vars nor config file are set', async () => { + mockConfigFile({}); + const result = await getConfigSettings('/config.json'); + expect(result.reindexIntervalMs).toBe(DEFAULT_CONFIG_SETTINGS.reindexIntervalMs); + expect(result.resyncConnectionIntervalMs).toBe(DEFAULT_CONFIG_SETTINGS.resyncConnectionIntervalMs); + expect(result.reindexRepoPollingIntervalMs).toBe(DEFAULT_CONFIG_SETTINGS.reindexRepoPollingIntervalMs); + expect(result.resyncConnectionPollingIntervalMs).toBe(DEFAULT_CONFIG_SETTINGS.resyncConnectionPollingIntervalMs); + }); + }); }); diff --git a/packages/shared/src/utils.ts b/packages/shared/src/utils.ts index 562674e84..dab4ccd8f 100644 --- a/packages/shared/src/utils.ts +++ b/packages/shared/src/utils.ts @@ -75,24 +75,27 @@ export const loadJsonFile = async ( export const getConfigSettings = async (configPath?: string): Promise => { - if (!configPath) { - return DEFAULT_CONFIG_SETTINGS; - } - - const config = await loadConfig(configPath); + const config = configPath ? await loadConfig(configPath) : undefined; + // Merge settings: env vars > config file > defaults + // Priority order: env var > config file `settings` block > hardcoded default return { ...DEFAULT_CONFIG_SETTINGS, - ...config.settings, + ...config?.settings, // Fall back to deprecated experiment_ variants if new keys are not set. repoDrivenPermissionSyncIntervalMs: - config.settings?.repoDrivenPermissionSyncIntervalMs - ?? config.settings?.experiment_repoDrivenPermissionSyncIntervalMs + config?.settings?.repoDrivenPermissionSyncIntervalMs + ?? config?.settings?.experiment_repoDrivenPermissionSyncIntervalMs ?? DEFAULT_CONFIG_SETTINGS.repoDrivenPermissionSyncIntervalMs, userDrivenPermissionSyncIntervalMs: - config.settings?.userDrivenPermissionSyncIntervalMs - ?? config.settings?.experiment_userDrivenPermissionSyncIntervalMs + config?.settings?.userDrivenPermissionSyncIntervalMs + ?? config?.settings?.experiment_userDrivenPermissionSyncIntervalMs ?? DEFAULT_CONFIG_SETTINGS.userDrivenPermissionSyncIntervalMs, + // Env var overrides (highest priority) + ...(env.REINDEX_INTERVAL_MS !== undefined && { reindexIntervalMs: env.REINDEX_INTERVAL_MS }), + ...(env.RESYNC_CONNECTION_INTERVAL_MS !== undefined && { resyncConnectionIntervalMs: env.RESYNC_CONNECTION_INTERVAL_MS }), + ...(env.REINDEX_REPO_POLLING_INTERVAL_MS !== undefined && { reindexRepoPollingIntervalMs: env.REINDEX_REPO_POLLING_INTERVAL_MS }), + ...(env.RESYNC_CONNECTION_POLLING_INTERVAL_MS !== undefined && { resyncConnectionPollingIntervalMs: env.RESYNC_CONNECTION_POLLING_INTERVAL_MS }), } }