[pull] master from botpress:master

integrations/github/integration.definition.ts

@@ -1,13 +1,14 @@
 import * as sdk from '@botpress/sdk'
 import { sentry as sentryHelpers } from '@botpress/sdk-addons'
+import filesReadonly from './bp_modules/files-readonly'
 
 import { INTEGRATION_NAME } from './src/const'
 import { actions, events, configuration, configurations, channels, user, secrets, states } from './src/definitions'
 
 export default new sdk.IntegrationDefinition({
   name: INTEGRATION_NAME,
   title: 'GitHub',
-  version: '1.1.10',
+  version: '1.2.0',
   icon: 'icon.svg',
   readme: 'hub.md',
   description: 'Manage GitHub issues, pull requests, and repositories.',
@@ -25,4 +26,16 @@ export default new sdk.IntegrationDefinition({
   attributes: {
     category: 'Developer Tools',
   },
-})
+}).extend(filesReadonly, ({}) => ({
+  entities: {},
+  actions: {
+    listItemsInFolder: {
+      name: 'filesReadonlyListItemsInFolder',
+      attributes: { ...sdk.WELL_KNOWN_ATTRIBUTES.HIDDEN_IN_STUDIO },
+    },
+    transferFileToBotpress: {
+      name: 'filesReadonlyTransferFileToBotpress',
+      attributes: { ...sdk.WELL_KNOWN_ATTRIBUTES.HIDDEN_IN_STUDIO },
+    },
+  },
+}))

integrations/github/package.json

@@ -21,5 +21,8 @@
     "@botpress/client": "workspace:*",
     "@botpress/sdk": "workspace:*",
     "@sentry/cli": "^2.39.1"
+  },
+  "bpDependencies": {
+    "files-readonly": "../../interfaces/files-readonly"
   }
 }

integrations/github/src/events/push/push-received.ts

@@ -0,0 +1,136 @@
+import type { PushEvent } from '@octokit/webhooks-types'
+import * as mapping from '../../files-readonly/mapping'
+import * as bp from '.botpress'
+
+const MAX_BATCH_SIZE = 50
+
+export const firePushReceived = async ({
+  githubEvent,
+  client,
+  logger,
+}: bp.HandlerProps & { githubEvent: PushEvent }) => {
+  try {
+    const repo = githubEvent.repository
+    const owner = repo.owner.login ?? repo.owner.name
+    const repoName = repo.name
+    const defaultRef = `refs/heads/${repo.default_branch ?? repo.master_branch ?? 'main'}`
+
+    if (githubEvent.ref !== defaultRef) {
+      logger.forBot().debug(`Ignoring push to non-default branch: ${githubEvent.ref}`)
+      return
+    }
+
+    const fileStates = new Map<string, 'created' | 'updated' | 'deleted'>()
+
+    for (const commit of githubEvent.commits) {
+      for (const filePath of commit.added ?? []) {
+        const prev = fileStates.get(filePath)
+        fileStates.set(filePath, prev === 'deleted' ? 'updated' : 'created')
+      }
+      for (const filePath of commit.modified ?? []) {
+        const prev = fileStates.get(filePath)
+        if (prev !== 'created') {
+          fileStates.set(filePath, 'updated')
+        }
+      }
+      for (const filePath of commit.removed ?? []) {
+        const prev = fileStates.get(filePath)
+        if (prev === 'created') {
+          fileStates.delete(filePath)
+        } else {
+          fileStates.set(filePath, 'deleted')
+        }
+      }
+    }
+
+    if (fileStates.size === 0) {
+      return
+    }
+
+    const created: bp.events.Events['aggregateFileChanges']['modifiedItems']['created'] = []
+    const updated: bp.events.Events['aggregateFileChanges']['modifiedItems']['updated'] = []
+    const deleted: bp.events.Events['aggregateFileChanges']['modifiedItems']['deleted'] = []
+
+    for (const [filePath, state] of fileStates) {
+      const file = mapping.mapPushFileToFile(owner, repoName, filePath)
+      if (state === 'created') {
+        created.push(file)
+      } else if (state === 'updated') {
+        updated.push(file)
+      } else {
+        deleted.push(file)
+      }
+    }
+
+    await _emitFileChangeEvents({ client, logger, changes: { created, updated, deleted } })
+  } catch (err: unknown) {
+    const error = err instanceof Error ? err : new Error(String(err))
+    logger.forBot().error('Failed to process push event; swallowing to prevent webhook retries', error)
+  }
+}
+
+const _emitFileChangeEvents = async ({
+  client,
+  logger,
+  changes,
+}: {
+  client: bp.Client
+  logger: bp.Logger
+  changes: FileChanges
+}) => {
+  try {
+    for (const batch of _getBatches(changes)) {
+      await client.createEvent({
+        type: 'aggregateFileChanges',
+        payload: {
+          modifiedItems: batch,
+        },
+      })
+    }
+  } catch (err: unknown) {
+    const error = err instanceof Error ? err : new Error(String(err))
+    logger.forBot().error('Failed to emit file-change events; swallowing to prevent webhook retries', error)
+  }
+}
+
+type FileChanges = {
+  created: bp.events.Events['aggregateFileChanges']['modifiedItems']['created']
+  updated: bp.events.Events['aggregateFileChanges']['modifiedItems']['updated']
+  deleted: bp.events.Events['aggregateFileChanges']['modifiedItems']['deleted']
+}
+
+function* _getBatches(changes: FileChanges): Generator<FileChanges> {
+  let createdIdx = 0
+  let updatedIdx = 0
+  let deletedIdx = 0
+
+  while (
+    createdIdx < changes.created.length ||
+    updatedIdx < changes.updated.length ||
+    deletedIdx < changes.deleted.length
+  ) {
+    const batch: FileChanges = { created: [], updated: [], deleted: [] }
+    let size = 0
+
+    while (size < MAX_BATCH_SIZE) {
+      const startSize = size
+      if (createdIdx < changes.created.length && size < MAX_BATCH_SIZE) {
+        batch.created.push(changes.created[createdIdx++]!)
+        size++
+      }
+      if (updatedIdx < changes.updated.length && size < MAX_BATCH_SIZE) {
+        batch.updated.push(changes.updated[updatedIdx++]!)
+        size++
+      }
+      if (deletedIdx < changes.deleted.length && size < MAX_BATCH_SIZE) {
+        batch.deleted.push(changes.deleted[deletedIdx++]!)
+        size++
+      }
+      if (size === startSize) {
+        break
+      }
+    }
+
+    yield batch
+  }
+}

integrations/github/src/files-readonly/actions/index.ts

@@ -0,0 +1,2 @@
+export { filesReadonlyListItemsInFolder } from './list-items-in-folder'
+export { filesReadonlyTransferFileToBotpress } from './transfer-file-to-botpress'

integrations/github/src/files-readonly/actions/list-items-in-folder.ts

@@ -0,0 +1,146 @@
+import { wrapActionAndInjectOctokit } from 'src/misc/action-wrapper'
+import * as mapping from '../mapping'
+
+const REPOS_PAGE_SIZE = 100
+const CONTENTS_PAGE_SIZE = 100
+
+export const filesReadonlyListItemsInFolder = wrapActionAndInjectOctokit(
+  { actionName: 'filesReadonlyListItemsInFolder', errorMessage: 'Failed to list items in folder' },
+  async ({ octokit, owner }, { folderId, filters, nextToken: prevToken }) => {
+    if (!folderId) {
+      return await _listRepos({ octokit, owner, prevToken, filters })
+    }
+
+    const repoInfo = mapping.decodeRepoFolderId(folderId)
+    if (repoInfo) {
+      return await _listRepoRoot({ octokit, ...repoInfo, prevToken, filters })
+    }
+
+    const contentInfo = mapping.decodeContentId(folderId)
+    if (contentInfo) {
+      return await _listFolderContents({ octokit, ...contentInfo, parentId: folderId, filters, prevToken })
+    }
+
+    return { items: [], meta: { nextToken: undefined } }
+  }
+)
+
+const _listRepos = async ({
+  octokit,
+  owner,
+  prevToken,
+  filters,
+}: {
+  octokit: { rest: any }
+  owner: string
+  prevToken?: string
+  filters?: { itemType?: string; maxSizeInBytes?: number; modifiedAfter?: string }
+}) => {
+  if (filters?.itemType === 'file') {
+    return { items: [], meta: { nextToken: undefined } }
+  }
+
+  const page = prevToken ? parseInt(prevToken, 10) : 1
+
+  let repos: mapping.GitHubRepo[]
+  try {
+    const response = await octokit.rest.repos.listForOrg({
+      org: owner,
+      per_page: REPOS_PAGE_SIZE,
+      page,
+      type: 'all',
+    })
+    repos = response.data as mapping.GitHubRepo[]
+  } catch (err: unknown) {
+    if (err && typeof err === 'object' && 'status' in err && err.status === 404) {
+      const response = await octokit.rest.repos.listForAuthenticatedUser({
+        per_page: REPOS_PAGE_SIZE,
+        page,
+      })
+      repos = response.data as mapping.GitHubRepo[]
+    } else {
+      throw err
+    }
+  }
+
+  const items = repos.map(mapping.mapRepoToFolder)
+
+  const hasMore = repos.length === REPOS_PAGE_SIZE
+  return {
+    items,
+    meta: { nextToken: hasMore ? String(page + 1) : undefined },
+  }
+}
+
+const _listRepoRoot = async ({
+  octokit,
+  owner,
+  repo,
+  prevToken,
+  filters,
+}: {
+  octokit: { rest: any }
+  owner: string
+  repo: string
+  prevToken?: string
+  filters?: { itemType?: string; maxSizeInBytes?: number; modifiedAfter?: string }
+}) => {
+  return _listFolderContents({
+    octokit,
+    owner,
+    repo,
+    path: '',
+    parentId: mapping.encodeRepoFolderId(owner, repo),
+    filters,
+    prevToken,
+  })
+}
+
+const _listFolderContents = async ({
+  octokit,
+  owner,
+  repo,
+  path,
+  parentId,
+  filters,
+  prevToken,
+}: {
+  octokit: { rest: any }
+  owner: string
+  repo: string
+  path: string
+  parentId: string
+  filters?: { itemType?: string; maxSizeInBytes?: number; modifiedAfter?: string }
+  prevToken?: string
+}) => {
+  const response = await octokit.rest.repos.getContent({
+    owner,
+    repo,
+    path: path || '',
+  })
+
+  const contents = Array.isArray(response.data) ? response.data : [response.data]
+  const contentItems = contents as mapping.GitHubContentItem[]
+
+  const startIndex = prevToken ? parseInt(prevToken, 10) : 0
+  const pageItems = contentItems.slice(startIndex, startIndex + CONTENTS_PAGE_SIZE)
+
+  let mappedItems = pageItems.map((item) => mapping.mapContentItemToItem(owner, repo, item, parentId))
+
+  if (filters?.itemType) {
+    const filterType = filters.itemType === 'folder' ? 'folder' : 'file'
+    mappedItems = mappedItems.filter((item) => item.type === filterType)
+  }
+
+  if (filters?.maxSizeInBytes) {
+    mappedItems = mappedItems.filter(
+      (item) => item.type !== 'file' || !item.sizeInBytes || item.sizeInBytes <= filters.maxSizeInBytes!
+    )
+  }
+
+  const hasMore = startIndex + CONTENTS_PAGE_SIZE < contentItems.length
+  return {
+    items: mappedItems,
+    meta: { nextToken: hasMore ? String(startIndex + CONTENTS_PAGE_SIZE) : undefined },
+  }
+}

integrations/github/src/files-readonly/actions/transfer-file-to-botpress.ts

@@ -0,0 +1,32 @@
+import * as sdk from '@botpress/sdk'
+import { wrapActionAndInjectOctokit } from 'src/misc/action-wrapper'
+import * as mapping from '../mapping'
+
+export const filesReadonlyTransferFileToBotpress = wrapActionAndInjectOctokit(
+  { actionName: 'filesReadonlyTransferFileToBotpress', errorMessage: 'Failed to transfer file to Botpress' },
+  async ({ octokit, client }, { file, fileKey, shouldIndex }) => {
+    const contentInfo = mapping.decodeContentId(file.id)
+    if (!contentInfo) {
+      throw new sdk.RuntimeError(`Invalid file ID: ${file.id}`)
+    }
+
+    const { owner, repo, path } = contentInfo
+
+    const response = await octokit.rest.repos.getContent({
+      owner,
+      repo,
+      path,
+      mediaType: { format: 'raw' },
+    })
+
+    const content = typeof response.data === 'string' ? response.data : JSON.stringify(response.data)
+
+    const { file: uploadedFile } = await client.uploadFile({
+      key: fileKey,
+      content,
+      index: shouldIndex,
+    })
+
+    return { botpressFileId: uploadedFile.id }
+  }
+)