index.ts

import AdminForth, { AdminForthPlugin, Filters, suggestIfTypo, AdminForthDataTypes, RAMLock, filtersTools, AdminForthFilterOperators } from "adminforth";
import type { IAdminForth, IHttpServer, AdminForthComponentDeclaration, AdminForthResourceColumn, AdminForthResource, BeforeLoginConfirmationFunction, AdminForthConfigMenuItem, AdminUser } from "adminforth";
import type { PluginOptions, SupportedLanguage } from './types.js';
import iso6391 from 'iso-639-1';
import { iso31661Alpha2ToAlpha3 } from 'iso-3166';
import path from 'path';
import fs from 'fs-extra';
import chokidar from 'chokidar';
import  { AsyncQueue } from '@sapphire/async-queue';
import getFlagEmoji from 'country-flag-svg';
import { parse } from 'bcp-47';
import pLimit from 'p-limit';
import { randomUUID } from 'crypto';
import { afLogger } from "adminforth";

const processFrontendMessagesQueue = new AsyncQueue();

const SLAVIC_PLURAL_EXAMPLES = {
  uk: 'яблук | Яблуко | Яблука | Яблук', // zero | singular | 2-4 | 5+
  bg: 'ябълки | ябълка | ябълки | ябълки', // zero | singular | 2-4 | 5+
  cs: 'jablek | jablko | jablka | jablek', // zero | singular | 2-4 | 5+
  hr: 'jabuka | jabuka | jabuke | jabuka', // zero | singular | 2-4 | 5+
  mk: 'јаболка | јаболко | јаболка | јаболка', // zero | singular | 2-4 | 5+
  pl: 'jabłek | jabłko | jabłka | jabłek', // zero | singular | 2-4 | 5+
  sk: 'jabĺk | jablko | jablká | jabĺk', // zero | singular | 2-4 | 5+
  sl: 'jabolk | jabolko | jabolka | jabolk', // zero | singular | 2-4 | 5+
  sr: 'јабука | јабука | јабуке | јабука', // zero | singular | 2-4 | 5+
  be: 'яблыкаў | яблык | яблыкі | яблыкаў', // zero | singular | 2-4 | 5+
  ru: 'яблок | яблоко | яблока | яблок', // zero | singular | 2-4 | 5+
};

const countryISO31661ByLangISO6391 = {
  en: 'us', // English → United States
  zh: 'cn', // Chinese → China
  hi: 'in', // Hindi → India
  ar: 'sa', // Arabic → Saudi Arabia
  ko: 'kr', // Korean → South Korea
  ja: 'jp', // Japanese → Japan
  uk: 'ua', // Ukrainian → Ukraine
  ur: 'pk', // Urdu → Pakistan
  sr: 'rs', // Serbian → Serbia
  da: 'dk' // Danish → Denmark
};

function getCountryCodeFromLangCode(lang: SupportedLanguage) {
  const [langCode, region] = String(lang).split('-');
  if (region && /^[A-Z]{2}$/.test(region)) {
    return region.toLowerCase();
  }
  return countryISO31661ByLangISO6391[langCode] || langCode;
}

function getPrimaryLanguageCode(langCode: SupportedLanguage): string {
  return String(langCode).split('-')[0];
}

function isValidSupportedLanguageTag(langCode: SupportedLanguage): boolean {
  try {
    const schema = parse(String(langCode), 
      { 
        normalize: true, 
        warning: (reason, code, offset) => {
          console.warn(`Warning in validating language tag ${langCode}: reason=${reason}, code=${code}, offset=${offset}`);
        }
      }
    );
    const isValid = schema.language.length === 2;
    return isValid;
  } catch (e) {
    return false;
  }
}

interface IFailedTranslation {
  lang: SupportedLanguage;
  en_string: string;
  failedReason: string;
}


interface ICachingAdapter {
  get(key: string): Promise<any>;
  set(key: string, value: any): Promise<void>;
  clear(key: string): Promise<void>;
}

class CachingAdapterMemory implements ICachingAdapter {
  cache: Record<string, any> = {};
  async get(key: string) {
    return this.cache[key];
  }
  async set(key: string, value: any) {
    this.cache[key] = value;
  }
  async clear(key: string) {
    if (this.cache[key]) {
      delete this.cache[key];
    }
  }
}

function ensureTemplateHasAllParams(template, newTemplate) {
  // string ensureTemplateHasAllParams("a {b} c {d}", "я {b} і {d} в") // true
  // string ensureTemplateHasAllParams("a {b} c {d}", "я і {d} в") // false
  // string ensureTemplateHasAllParams("a {b} c {d}", "я {bb} і {d} в") // false
  const existingParams = template.match(/{[^}]+}/g);
  const newParams = newTemplate.match(/{[^}]+}/g);
  const existingParamsSet = new Set(existingParams);
  const newParamsSet = new Set(newParams);
  return existingParamsSet.size === newParamsSet.size && [...existingParamsSet].every(p => newParamsSet.has(p));
}

class AiTranslateError extends Error {
  constructor(message: string) {
    super(message);
    this.name = 'AiTranslateError';
  }
}
export default class I18nPlugin extends AdminForthPlugin {
  options: PluginOptions;
  emailField: AdminForthResourceColumn;
  passwordField: AdminForthResourceColumn;
  authResource: AdminForthResource;
  emailConfirmedField?: AdminForthResourceColumn;
  trFieldNames: Partial<Record<SupportedLanguage, string>>;
  enFieldName: string;
  cache: ICachingAdapter;
  primaryKeyFieldName: string;
  menuItemWithBadgeId: string;

  adminforth: IAdminForth;

  externalAppOnly: boolean;

  // sorted by name list of all supported languages, without en e.g. 'al|ro|uk'
  fullCompleatedFieldValue: string;
  primaryLanguage: SupportedLanguage;

  constructor(options: PluginOptions) {
    super(options, import.meta.url);
    this.options = options;
    this.cache = new CachingAdapterMemory();
    this.trFieldNames = {};
    this.primaryLanguage = options.primaryLanguage || 'en';
  }

  async computeCompletedFieldValue(record: any) {
    return this.options.supportedLanguages.reduce((acc: string, lang: SupportedLanguage): string => {
      if (lang === 'en') {
        return acc;
      }
      if (record[this.trFieldNames[lang]]) {
        if (acc !== '') {
          acc += '|';
        }
        acc += (lang as string);
      }
      return acc;
    }, '');
  }

  async updateUntranslatedMenuBadge() {
    if (this.menuItemWithBadgeId) {
      const resource = this.adminforth.resource(this.resourceConfig.resourceId);
      const count = await resource.count([Filters.NEQ(this.options.completedFieldName, this.fullCompleatedFieldValue)]);

      this.adminforth.websocket.publish(`/opentopic/update-menu-badge/${this.menuItemWithBadgeId}`, {
        badge: count || null
      });
    }
  }


  async modifyResourceConfig(adminforth: IAdminForth, resourceConfig: AdminForthResource) {
    super.modifyResourceConfig(adminforth, resourceConfig);

    if (!this.adminforth.config.componentsToExplicitRegister) {
      this.adminforth.config.componentsToExplicitRegister = [];
    }
    this.adminforth.config.componentsToExplicitRegister.push(
      {
        file: this.componentPath('TranslationJobViewComponent.vue')
      }
    );

    // validate each supported language: ISO 639-1 or BCP-47 with region (e.g., en-GB)
    this.options.supportedLanguages.forEach((lang) => {
      if (!isValidSupportedLanguageTag(lang)) {
        throw new Error(`Invalid language code ${lang}. Use ISO 639-1 (e.g., 'en') or BCP-47 with region (e.g., 'en-GB').`);
      }
    });

    if (this.options.translateLangAsBCP47Code) {
      for (const [lang, bcp47] of Object.entries(this.options.translateLangAsBCP47Code)) {
        if (!this.options.supportedLanguages.includes(lang as SupportedLanguage)) {
          throw new Error(`Invalid language code ${lang} in translateLangAsBCP47Code. It must be one of the supportedLanguages.`);
        }
      }
    }

    this.externalAppOnly = this.options.externalAppOnly === true;

    // find primary key field
    this.primaryKeyFieldName = resourceConfig.columns.find(c => c.primaryKey)?.name;

    if (!this.primaryKeyFieldName) {
      throw new Error(`Primary key field not found in resource ${resourceConfig.resourceId}`);
    }

    // parse trFieldNames
    for (const lang of this.options.supportedLanguages) {
      if (lang === 'en') {
        continue;
      }
      if (this.options.translationFieldNames?.[lang]) {
        this.trFieldNames[lang] = this.options.translationFieldNames[lang];
      } else {
        this.trFieldNames[lang] = lang + '_string';
      }
      // find column by name
      const column = resourceConfig.columns.find(c => c.name === this.trFieldNames[lang]);
      if (!column) {
        throw new Error(`Field ${this.trFieldNames[lang]} not found for storing translation for language ${lang}
           in resource ${resourceConfig.resourceId}, consider adding it to columns or change trFieldNames option to remap it to existing column`);
      }

      column.components = column.components || {};
      
      // set edit and create custom component - SingleMultiInput.vue
      column.components.edit = {
        file: this.componentPath('SingleMultiInput.vue'),
      };
      column.components.create = {
        file: this.componentPath('SingleMultiInput.vue'),
      };

      // set ListCell for list
      column.components.list = {
        file: this.componentPath('ListCell.vue'),
        meta: {
          pluginInstanceId: this.pluginInstanceId,
          lang,
          reviewedCheckboxesFieldName: this.options.reviewedCheckboxesFieldName,
          enFieldName: this.enFieldName,
        },
      };
    }

    this.enFieldName = this.trFieldNames['en'] || 'en_string';

    this.fullCompleatedFieldValue = this.options.supportedLanguages.reduce((acc: string, lang: SupportedLanguage) => {
      if (lang === 'en') {
        return acc;
      }
      if (acc !== '') {
        acc += '|';
      }
      acc += lang as string;
      return acc;
    }, '');

    // if not enFieldName column is not found, throw error
    const enColumn = resourceConfig.columns.find(c => c.name === this.enFieldName);
    if (!enColumn) {
      throw new Error(`Field ${this.enFieldName} not found column to store english original string in resource ${resourceConfig.resourceId}`);
    }

    enColumn.components = enColumn.components || {};
    enColumn.components.edit = {
      file: this.componentPath('SingleMultiInput.vue'),
    };
    enColumn.components.create = {
      file: this.componentPath('SingleMultiInput.vue'),
    };
    enColumn.components.list = {
      file: this.componentPath('ListCell.vue'),
      meta: {
        enFieldName: this.enFieldName,
      }
    };
    enColumn.editReadonly = true;

    // if sourceFieldName defined, check it exists
    if (this.options.sourceFieldName) {
      if (!resourceConfig.columns.find(c => c.name === this.options.sourceFieldName)) {
        throw new Error(`Field ${this.options.sourceFieldName} not found in resource ${resourceConfig.resourceId}`);
      }
    }

    // if completedFieldName defined, check it exists and should be string
    if (this.options.completedFieldName) {
      const column = resourceConfig.columns.find(c => c.name === this.options.completedFieldName);
      if (!column) {
        const similar = suggestIfTypo(resourceConfig.columns.map((col) => col.name), this.options.completedFieldName);
        throw new Error(`Field ${this.options.completedFieldName} not found in resource ${resourceConfig.resourceId}${similar ? `Did you mean '${similar}'?` : ''}`);
      }

      // if showIn is not defined, add it as empty
      column.showIn = {
        show: false,
        list: false,
        edit: false,
        create: false,
        filter: false,
      };

      // add virtual field for incomplete
      resourceConfig.columns.unshift({
        name: 'fully_translated',
        label: 'Fully translated',
        virtual: true,
        filterOptions: {
          multiselect: false,
        },
        showIn: {
          show: true,
          list: true,
          edit: false,
          create: false,
          filter: true,
        },
        type: AdminForthDataTypes.BOOLEAN,
      });
    }

    const compMeta = { 
      brandSlug: adminforth.config.customization.brandNameSlug,
      pluginInstanceId: this.pluginInstanceId,
      primaryLanguage: this.primaryLanguage,
      afOrder: this.options.loginPageLanguageSelectorOrder || 0,
      supportedLanguages: this.options.supportedLanguages.map(lang => (
        {
          code: lang,
          // lang name on on language native name
          name: iso6391.getNativeName(getPrimaryLanguageCode(lang)),
        }
      ))
    };
    // add underLogin component
    if (!this.externalAppOnly) {
      (adminforth.config.customization.loginPageInjections.underLoginButton as Array<any>).push({ 
        file: this.componentPath('LanguageUnderLogin.vue'),
        meta: compMeta
      });

      (adminforth.config.customization.globalInjections.userMenu).push({
        file: this.componentPath('LanguageInUserMenu.vue'),
        meta: compMeta
      });

      adminforth.config.customization.globalInjections.everyPageBottom.push({
        file: this.componentPath('LanguageEveryPageLoader.vue'),
        meta: compMeta
      });
    }

    // disable create allowedActions for translations
    resourceConfig.options.allowedActions.create = false;

    // add hook to validate user did not screw up with template params
    resourceConfig.hooks.edit.beforeSave.push(async ({ updates, oldRecord }: { updates: any, oldRecord?: any }): Promise<{ ok: boolean, error?: string }> => {
      for (const lang of this.options.supportedLanguages) {
        if (lang === 'en') {
          continue;
        }
        if (updates[this.trFieldNames[lang]]) {  // if user set '', it will have '' in updates, then it is fine, we shoudl nto check it
          if (!ensureTemplateHasAllParams(oldRecord[this.enFieldName], updates[this.trFieldNames[lang]])) {
            return { ok: false, error: `Template params mismatch for ${updates[this.enFieldName]}. Template param names should be the same as in original string. E. g. 'Hello {name}', should be 'Hola {name}' and not 'Hola {nombre}'!` };
          }
        }
      }
      return { ok: true };
    });

    // add hook on edit of any translation
    resourceConfig.hooks.edit.afterSave.push(async ({ updates, oldRecord }: { updates: any, oldRecord?: any }): Promise<{ ok: boolean, error?: string }> => {
      if (oldRecord) {
        // find lang which changed
        let langsChanged: SupportedLanguage[] = [];
        for (const lang of this.options.supportedLanguages) {
          if (lang === 'en') {
            continue;
          }
          if (updates[this.trFieldNames[lang]] !== undefined) {
            langsChanged.push(lang);
          }
        }
        
        // clear frontend cache for all langsChanged
        for (const lang of langsChanged) {
          this.cache.clear(`${this.resourceConfig.resourceId}:${oldRecord[this.options.categoryFieldName]}:${lang}`);
          this.cache.clear(`${this.resourceConfig.resourceId}:${oldRecord[this.options.categoryFieldName]}:${lang}:${oldRecord[this.enFieldName]}`);
        }
        this.updateUntranslatedMenuBadge();
      }
      // clear frontend cache for all lan
      return { ok: true };
    });

    // add hook on delete of any translation to reset cache
    resourceConfig.hooks.delete.afterSave.push(async ({ record }: { record: any }): Promise<{ ok: boolean, error?: string }> => {
      for (const lang of this.options.supportedLanguages) {
        // if frontend, clear frontend cache
        this.cache.clear(`${this.resourceConfig.resourceId}:${record[this.options.categoryFieldName]}:${lang}`);
        this.cache.clear(`${this.resourceConfig.resourceId}:${record[this.options.categoryFieldName]}:${lang}:${record[this.enFieldName]}`);
      }
      this.updateUntranslatedMenuBadge();
      return { ok: true };
    });

    if (this.options.completedFieldName) {
      // on show and list add a list hook which will add incomplete field to record if translation is missing for at least one language
      const addIncompleteField = (record: any) => {
        // form list of all langs, sorted by alphabet, without en, to get 'al|ro|uk'
        
      
        record.fully_translated = this.fullCompleatedFieldValue === record[this.options.completedFieldName];
      }
      resourceConfig.hooks.list.afterDatasourceResponse.push(async ({ response }: { response: any[] }): Promise<{  ok: boolean, error?: string }> => {
        response.forEach(addIncompleteField);
        return { ok: true }
      });
      resourceConfig.hooks.show.afterDatasourceResponse.push(async ({ response }: { response: any }): Promise<{  ok: boolean, error?: string }> => {
        addIncompleteField(response.length && response[0]);
        return { ok: true }
      });

      // also add edit hook beforeSave to update completedFieldName
      resourceConfig.hooks.edit.beforeSave.push(async ({ record, oldRecord }: { record: any, oldRecord: any }): Promise<{ ok: boolean, error?: string }> => {
        const futureRecord = { ...oldRecord, ...record };
        
        const futureCompletedFieldValue = await this.computeCompletedFieldValue(futureRecord);
    
        record[this.options.completedFieldName] = futureCompletedFieldValue;
        return { ok: true };
      });

      // add list hook to support filtering by fully_translated virtual field
      resourceConfig.hooks.list.beforeDatasourceRequest.push(async ({ query }: { query: any }): Promise<{ ok: boolean, error?: string }> => {
        if (!query.filters || query.filters.length === 0) {
          query.filters = [];
        }

        // get fully_translated field from filter if it is there
        const fullyTranslatedFilter = query.filters.find((f: any) => f.field === 'fully_translated');
        if (fullyTranslatedFilter) {
          // remove it from filters because it is virtual field
          query.filters = query.filters.filter((f: any) => f.field !== 'fully_translated');
          if (fullyTranslatedFilter.value) {
            query.filters.push({
              field: this.options.completedFieldName,
              value: this.fullCompleatedFieldValue,
              operator: 'eq',
            });
          } else {
            query.filters.push({
              field: this.options.completedFieldName,
              value: this.fullCompleatedFieldValue,
              operator: 'ne',
            });
          }
        }

        return { ok: true };
      });
    }

    // add bulk action

    const pageInjection = {
      file: this.componentPath('BulkActionButton.vue'),
      meta: {
        supportedLanguages: this.options.supportedLanguages,
        pluginInstanceId: this.pluginInstanceId,
      }
    }

    if (!resourceConfig.options.pageInjections) {
      resourceConfig.options.pageInjections = {};
    }
    if (!resourceConfig.options.pageInjections.list) {
      resourceConfig.options.pageInjections.list = {};
    }
    if (!resourceConfig.options.pageInjections.list.threeDotsDropdownItems) {
      resourceConfig.options.pageInjections.list.threeDotsDropdownItems = [];
    }

    (resourceConfig.options.pageInjections.list.threeDotsDropdownItems as AdminForthComponentDeclaration[]).push(pageInjection);

    // if there is menu item with resourceId, add .badge function showing number of untranslated strings
    const addBadgeCountToMenuItem = (menuItem: AdminForthConfigMenuItem) => {
      this.menuItemWithBadgeId = menuItem.itemId;
      menuItem.badge = async () => {
        const resource = adminforth.resource(menuItem.resourceId);
        const count = await resource.count([Filters.NEQ(this.options.completedFieldName, this.fullCompleatedFieldValue)]);
        return count ? `${count}` : null;
      };
      menuItem.badgeTooltip = 'Untranslated count';
    }
    adminforth.config.menu.forEach((menuItem) => {
      if (menuItem.resourceId === resourceConfig.resourceId && !menuItem.badge) {
        addBadgeCountToMenuItem(menuItem);
      }
      if (menuItem.children) {
        menuItem.children.forEach((child) => {
          if (child.resourceId === resourceConfig.resourceId && !child.badge) {
            addBadgeCountToMenuItem(child);
          }
        });
      }
    });
  }

  async generateAndSaveBunch (
    prompt: string,
    strings: { en_string: string, category: string }[], 
    translations: any[],
    updateStrings: Record<string, { updates: any, category: string, strId: string, enStr: string, translatedStr: string }> = {},
    lang: string,
    failedToTranslate: IFailedTranslation[],
    needToTranslateByLang: Record<string, any> = {},
    jobId: string,
    promptCost: number,
  ): Promise<void>{

    // return [];
    const jsonSchemaProperties = {};
    strings.forEach(s => {
      jsonSchemaProperties[s.en_string] = { 
        type: 'string',
        minLength: 1,
      };
    });

    const jsonSchemaRequired = strings.map(s => s.en_string);
    const dedupRequired = Array.from(new Set(jsonSchemaRequired));
    // call OpenAI
    const resp = await this.options.completeAdapter.complete(
      prompt,
      [],
      prompt.length * 2,
      {
        json_schema: {
          name: "translation_response",
          schema: {
            type: "object",
            properties: jsonSchemaProperties,
            required: dedupRequired,
          },
        },
      }
    );
    
    process.env.HEAVY_DEBUG && console.log(`🪲🔪LLM resp >> ${prompt.length}, <<${resp.content.length} :\n\n`, JSON.stringify(resp));
    
    if (resp.error) {
      throw new AiTranslateError(resp.error);
    }

    let res;
    try {
      res = resp.content;
    } catch (e) {
      console.error(`Error in parsing LLM resp: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, );
      strings.forEach(s => {
        failedToTranslate.push({
          lang: lang as SupportedLanguage,
          en_string: s.en_string,
          failedReason: "Error in parsing LLM response"
        });
      });
      return null;
    }

    const backgroundJobsPlugin = this.adminforth.getPluginByClassName<any>('BackgroundJobsPlugin');

    backgroundJobsPlugin.updateJobFieldsAtomically(jobId, async () => {
      // do all set / get fields in this function to make state update atomic and there is no conflicts when 2 tasks in parallel do get before set.
      // don't do long awaits in this callback, since it has exclusive lock.
      let totalUsedTokens = await backgroundJobsPlugin.getJobField(jobId, 'totalUsedTokens');
      totalUsedTokens += promptCost;
      await backgroundJobsPlugin.setJobField(jobId, 'totalUsedTokens', totalUsedTokens);
    })


    try {
      res = JSON.parse(res);
    } catch (e) {
      console.error(`Error in parsing LLM resp json: ${resp}\n Prompt was: ${prompt}\n Resp was: ${JSON.stringify(resp)}`, );
      strings.forEach(s => {
        failedToTranslate.push({
          lang: lang as SupportedLanguage,
          en_string: s.en_string,
          failedReason: "Error in parsing LLM response JSON"
        });
      });
      return null;
    }


    for (const [enStr, translatedStr] of Object.entries(res) as [string, string][]) {
      const translationsTargeted = translations.filter(t => t[this.enFieldName] === enStr);
      // might be several with same en_string
      for (const translation of translationsTargeted) {
        //translation[this.trFieldNames[lang]] = translatedStr;
        if (!updateStrings[translation[this.primaryKeyFieldName]]) {

          updateStrings[translation[this.primaryKeyFieldName]] = {
            updates: {},
            translatedStr,
            enStr,
            category: translation[this.options.categoryFieldName],
            strId: translation[this.primaryKeyFieldName],
          };
        }
        // make sure LLM did not screw up with template params
        if (translation[this.enFieldName].includes('{') && !ensureTemplateHasAllParams(translation[this.enFieldName], translatedStr)) {
          console.warn(`LLM Screwed up with template params mismatch for "${translation[this.enFieldName]}"on language ${lang}, it returned "${translatedStr}"`);
          continue;
        }
        updateStrings[
          translation[this.primaryKeyFieldName]
        ].updates[this.trFieldNames[lang]] = translatedStr;
      }
    }

    const langsInvolved = new Set(Object.keys(needToTranslateByLang));

    // here we need to save updateStrings
    await Promise.all(
      Object.entries(updateStrings).map(
        async ([_, { updates, strId }]: [string, { updates: any, category: string, strId: string }]) => {
          // get old full record
          const oldRecord = await this.adminforth.resource(this.resourceConfig.resourceId).get([Filters.EQ(this.primaryKeyFieldName, strId)]);

          // because this will translate all languages, we can set completedLangs to all languages
          const futureCompletedFieldValue = await this.computeCompletedFieldValue({ ...oldRecord, ...updates });

          await this.adminforth.resource(this.resourceConfig.resourceId).update(strId, {
            ...updates,
            [this.options.completedFieldName]: futureCompletedFieldValue,
          });
        }
      )
    );
    

    for (const lang of langsInvolved) {
      const categoriesInvolved = new Set();
      for (const { enStr, category } of Object.values(updateStrings)) {
        categoriesInvolved.add(category);
        await this.cache.clear(`${this.resourceConfig.resourceId}:${category}:${lang}:${enStr}`);
      }
      for (const category of categoriesInvolved) {
        await this.cache.clear(`${this.resourceConfig.resourceId}:${category}:${lang}`);
      }
    }

  }

  async getTranslateToLangTasks (
    langIsoCode: SupportedLanguage, 
    strings: { id: string, en_string: string, category: string }[], 
    plurals=false,
    translations: any[],
    updateStrings: Record<string, { updates: any, category: string, strId: string, enStr: string, translatedStr: string }> = {},
    needToTranslateByLang: Record<string, any> = {},
    adminUser: AdminUser,
  ): Promise<any> {

    const maxInputTokens = this.options.inputTokensPerBatch ?? 30000;

    const limit = pLimit(30); 
    const enStringsTokenLengthCache: Record<string, any> = {};
    

    const tokenLengthPerString = async ({ str, id }: { str: string; id: string }): Promise<void> => {
      const objectToPush = {
        en_string: str,
        numOfTokens: await this.options.completeAdapter.measureTokensCount(`"${str}":"", \n`)
      };
      enStringsTokenLengthCache[id] = objectToPush;
    }
    const promises = strings.map(s => limit(() => tokenLengthPerString({ str: s.en_string, id: s.id })));

    await Promise.all(promises);
    if (strings.length === 0) {
      return;
    }

    const replacedLanguageCodeForTranslations = this.options.translateLangAsBCP47Code && langIsoCode.length === 2 ? this.options.translateLangAsBCP47Code[langIsoCode as any] : null;
    const langSchema = parse(String(langIsoCode), 
      { 
        normalize: true, 
      }
    );
    const langCode = replacedLanguageCodeForTranslations ? replacedLanguageCodeForTranslations : langIsoCode;
    const lang = langIsoCode;
    const primaryLang = getPrimaryLanguageCode(lang);
    const langName = iso6391.getName(primaryLang);
    const requestSlavicPlurals = Object.keys(SLAVIC_PLURAL_EXAMPLES).includes(primaryLang) && plurals;
    const region = langSchema.region;
    const basePrompt = `
      I need to translate strings in JSON to ${langName} language ${replacedLanguageCodeForTranslations || lang.length > 2 ? `BCP-47 code ${langCode}` : `ISO 639-1 code ${langIsoCode}`} from English for my web app.

      
      ${region ? `Use the regional conventions for ${langCode} (region ${region}), including spelling, punctuation, and formatting.` : ''}
      ${requestSlavicPlurals ? `You should provide 4 slavic forms (in format "zero count | singular count | 2-4 | 5+") e.g. "apple | apples" should become "${SLAVIC_PLURAL_EXAMPLES[lang]}"` : ''}
      Keep keys, as is, write translation into values! If keys have variables (in curly brackets), then translated strings should have them as well (variables itself should not be translated). Here are the strings:
      \`\`\`json
      {

      }
      \`\`\`
    `;

    const failedToTranslate: IFailedTranslation[] = [];
    const basePromptTokenLength = await this.options.completeAdapter.measureTokensCount(basePrompt);
    const allowedTokensAmountForFields = maxInputTokens - basePromptTokenLength;
    const stringsToTranslate: Record<string, { id: string; en_string: string; category: string }> = Object.fromEntries(strings.map(s => [s.id, s]));
    const generationTasksInitialData = []
    while (Object.keys(stringsToTranslate).length !== 0) {
      const stringBanch = [];
      const stringIdsInBatch = [];
      let banchTokens = 0;
      for (const id of Object.keys(stringsToTranslate)) {
        const { en_string } = stringsToTranslate[id];
        const numberOfTokensForString = enStringsTokenLengthCache[id]?.numOfTokens || 0;
        if( banchTokens + numberOfTokensForString <= allowedTokensAmountForFields ) {
          stringBanch.push( en_string );
          stringIdsInBatch.push(id);
          banchTokens += numberOfTokensForString;
        } else {
          continue;
        }
      }
      if ( stringBanch.length === 0 ) {
        Object.values(stringsToTranslate).forEach(s => {
          failedToTranslate.push({
            lang,
            en_string: s.en_string,
            failedReason: "Not enough input generation tokens"
          });
          generationTasksInitialData.push(
            { 
              state: {
                failedToTranslate,
              }
            }
          )
        });
        break;
      }

      for ( const id of stringIdsInBatch) {
        delete stringsToTranslate[id];
      }

      const promptToGenerate = basePrompt.split(`\`\`\`json`)[0] + 
        `\`\`\`json
        {
          ${
            stringBanch.map(s => `"${s}": ""`).join(",\n")
          }
        }
        \`\`\``;
      const stringBanchCopy = [...stringBanch];
      generationTasksInitialData.push(
        {
          state: {
            taskName: `Translate ${strings.length} strings`,
            prompt: promptToGenerate,
            strings: strings.filter(s => stringBanchCopy.includes(s.en_string)),
            translations: translations.filter(t => stringBanchCopy.includes(t.en_string)),
            updateStrings,
            lang,
            failedToTranslate,
            needToTranslateByLang,
            promptCost: basePromptTokenLength + banchTokens,
          }
        }
      ) 
    }

    return generationTasksInitialData;
  }


  // returns translated count
  async bulkTranslate({ selectedIds, selectedLanguages, adminUser }: 
    { 
      selectedIds: string[], 
      selectedLanguages?: SupportedLanguage[],
      adminUser: AdminUser,
    }): Promise<string>
    {
    const needToTranslateByLang : Partial<
      Record<
        SupportedLanguage,
        {
          id: string;
          en_string: string;
          category: string;
        }[]
      >
    > = {};

    const translations = await this.adminforth.resource(this.resourceConfig.resourceId).list(Filters.IN(this.primaryKeyFieldName, selectedIds));
    const languagesToProcess = selectedLanguages || this.options.supportedLanguages;
    for (const lang of languagesToProcess) {
      if (lang === 'en') {
        // all strings are in English, no need to translate
        continue;
      }
      for (const translation of translations) {
        if (!translation[this.trFieldNames[lang]]) {
          if (!needToTranslateByLang[lang]) {
            needToTranslateByLang[lang] = [];
          }
          needToTranslateByLang[lang].push({
            id: translation[this.primaryKeyFieldName],
            'en_string': translation[this.enFieldName],
            category: translation[this.options.categoryFieldName],
          })
        }
      }
    }

    const updateStrings: Record<string, { 
      updates: any, 
      category: string,
      strId: string,
      enStr: string,
      translatedStr: string
    }> = {};


    let generationTasksInitialData = [];
    await Promise.all(
      Object.entries(needToTranslateByLang).map(
        async ([lang, strings]: [SupportedLanguage, { id: string, en_string: string, category: string }[]]) => {
          // first translate without plurals
          const stringsWithoutPlurals = strings.filter(s => !s.en_string.includes('|'));
          const noPluralTranslationsTasks = await this.getTranslateToLangTasks(lang, stringsWithoutPlurals, false, translations, updateStrings, needToTranslateByLang, adminUser);

          const stringsWithPlurals = strings.filter(s => s.en_string.includes('|'));
          const pluralTranslationsTasks = await this.getTranslateToLangTasks(lang, stringsWithPlurals, true, translations, updateStrings, needToTranslateByLang, adminUser);
          generationTasksInitialData = generationTasksInitialData.concat(noPluralTranslationsTasks || []).concat(pluralTranslationsTasks || []);
        }
      )
    );


    const totalTranslationTokenCost = generationTasksInitialData.reduce((a, b) => a + (b.state?.promptCost || 0), 0);
    const backgroundJobsPlugin = this.adminforth.getPluginByClassName<any>('BackgroundJobsPlugin');
    const jobId = await backgroundJobsPlugin.startNewJob(
      `Translate ${selectedIds.length} items`, //job name
      adminUser, // adminuser
      generationTasksInitialData, //initial tasks
      'translation_job_handler', //job handler name
    );

    afLogger.info(`Started background job with ID ${jobId} `);
    await backgroundJobsPlugin.setJobField(jobId, 'totalTranslationTokenCost', totalTranslationTokenCost);
    await backgroundJobsPlugin.setJobField(jobId, 'totalUsedTokens', 0);

    return jobId
  }

  
  async processExtractedMessages(adminforth: IAdminForth, filePath: string) {
    await processFrontendMessagesQueue.wait();
    // messages file is in i18n-messages.json
    let messages;
    try {
      messages = await fs.readJson(filePath);
      process.env.HEAVY_DEBUG && console.info('🐛 Messages file found');

    } catch (e) {
      process.env.HEAVY_DEBUG && console.error('🐛 Messages file not yet exists, probably npm run i18n:extract not finished/started yet, might be ok');
      return;
    }
    // loop over missingKeys[i].path and add them to database if not exists
    const messagesForFeed = messages.missingKeys.map((mk) => {
      return {
        en_string: mk.path,
        source: mk.file,
      };
    });

    await this.feedCategoryTranslations(messagesForFeed, 'frontend')
    
    
  }


  async tryProcessAndWatch(adminforth: IAdminForth) {
    const serveDir = adminforth.codeInjector.getServeDir();
    // messages file is in i18n-messages.json
    const messagesFile = path.join(serveDir, 'i18n-messages.json');
    process.env.HEAVY_DEBUG && console.log('🪲🔔messagesFile read started', messagesFile);
    this.processExtractedMessages(adminforth, messagesFile);
    // we use watcher because file can't be yet created when we start - bundleNow can be done in build time or can be done now
    // that is why we make attempt to process it now and then watch for changes
    const w = chokidar.watch(messagesFile, { 
      persistent: true,
      ignoreInitial: true, // don't trigger 'add' event for existing file on start
    });
    w.on('change', () => {
      process.env.HEAVY_DEBUG && console.log('🪲🔔messagesFile change', messagesFile);
      this.processExtractedMessages(adminforth, messagesFile);
    });
    w.on('add', () => {
      process.env.HEAVY_DEBUG && console.log('🪲🔔messagesFile add', messagesFile);
      this.processExtractedMessages(adminforth, messagesFile);
    });
  }
  
  validateConfigAfterDiscover(adminforth: IAdminForth, resourceConfig: AdminForthResource) {
    // optional method where you can safely check field types after database discovery was performed
    // ensure each trFieldName (apart from enFieldName) is nullable column of type string
    const backgroundJobsPlugin = adminforth.getPluginByClassName<any>('BackgroundJobsPlugin');

    if (!backgroundJobsPlugin) {
      throw new Error(`BackgroundJobsPlugin is required for ${this.constructor.name} to work, please add it to your plugins`);
    }

    backgroundJobsPlugin.registerTaskHandler({
      // job handler name
      jobHandlerName: 'translation_job_handler',
      //handler function
      handler: async ({ jobId, setTaskStateField, getTaskStateField }) => {
        const initialState: {    
          taskName: string,
          prompt?: string,
          strings?: { en_string: string, category: string }[], 
          translations?: any[],
          updateStrings?: Record<string, { updates: any, category: string, strId: string, enStr: string, translatedStr: string }>,
          lang?: string,
          failedToTranslate?: IFailedTranslation[],
          needToTranslateByLang?: Record<string, any>,
          promptCost?: number,
        } = await getTaskStateField();

        if ( initialState.prompt && initialState.strings && initialState.translations && initialState.updateStrings && initialState.lang && initialState.failedToTranslate && initialState.needToTranslateByLang) {
          await this.generateAndSaveBunch(
            initialState.prompt,
            initialState.strings,
            initialState.translations,
            initialState.updateStrings,
            initialState.lang,
            initialState.failedToTranslate,
            initialState.needToTranslateByLang,
            jobId,
            initialState.promptCost
          );
        }
        afLogger.debug(`Translation task for language ${initialState.lang} completed.`);

        const stateToSave = {
          taskName: initialState.taskName,
          lang: initialState.lang,
          failedToTranslate: initialState.failedToTranslate,
        }
        await setTaskStateField(stateToSave);

        this.adminforth.websocket.publish('/translation_progress', {});
        if (initialState.failedToTranslate.length > 0) {
          afLogger.error(`Failed to translate some strings for language ${initialState.lang} in plugin ${this.constructor.name}:, ${initialState.failedToTranslate}`);
          throw new Error(`Failed to translate some strings for language ${initialState.lang}, check job details for more info`);
        }
      },
      //limit of tasks, that are running in parallel
      parallelLimit: this.options.parallelTranslationLimit || 20,
    })

    backgroundJobsPlugin.registerTaskDetailsComponent({
      jobHandlerName: 'translation_job_handler', // Handler name
      component: { 
        file: this.componentPath('TranslationJobViewComponent.vue')  //custom component for the job details
      },
    })


    if (this.options.completeAdapter) {
      this.options.completeAdapter.validate();
    }

    for (const lang of this.options.supportedLanguages) {
      if (lang === 'en') {
        continue;
      }
      const column = resourceConfig.columns.find(c => c.name === this.trFieldNames[lang]);
      if (!column) {
        throw new Error(`Field ${this.trFieldNames[lang]} not found for storing translation for language ${lang}
           in resource ${resourceConfig.resourceId}, consider adding it to columns or change trFieldNames option to remap it to existing column`);
      }
      if (column.required.create || column.required.edit) {
        throw new Error(`Field ${this.trFieldNames[lang]} should be not required in resource ${resourceConfig.resourceId}`);
      }
    }

    if (this.options.reviewedCheckboxesFieldName) {
      // ensure type is JSON
      const column = resourceConfig.columns.find(c => c.name === this.options.reviewedCheckboxesFieldName);
      if (!column) {
        const similar = suggestIfTypo(resourceConfig.columns.map((col) => col.name), this.options.reviewedCheckboxesFieldName);
        throw new Error(`Field ${this.options.reviewedCheckboxesFieldName} not found in resource ${resourceConfig.resourceId}${similar ? `Did you mean '${similar}'?` : ''}`);
      }
      if (column.type !== AdminForthDataTypes.JSON) {
        throw new Error(`Field ${this.options.reviewedCheckboxesFieldName} should be of type JSON in resource ${resourceConfig.resourceId}, but it is ${column.type}`);
      }
    }


    // ensure categoryFieldName defined and is string
    if (!this.options.categoryFieldName) {
      throw new Error(`categoryFieldName option is not defined. It is used to categorize translations and return only specific category e.g. to frontend`);
    }
    const categoryColumn = resourceConfig.columns.find(c => c.name === this.options.categoryFieldName);
    if (!categoryColumn) {
      throw new Error(`Field ${this.options.categoryFieldName} not found in resource ${resourceConfig.resourceId}`);
    }
    if (categoryColumn.type !== AdminForthDataTypes.STRING && categoryColumn.type !== AdminForthDataTypes.TEXT) {
      throw new Error(`Field ${this.options.categoryFieldName} should be of type string in resource ${resourceConfig.resourceId}, but it is ${categoryColumn.type}`);
    }

    // in this plugin we will use plugin to fill the database with missing language messages
    if (!this.externalAppOnly) {
      this.tryProcessAndWatch(adminforth);

      adminforth.tr = async (msg: string | null | undefined, category: string, lang: string, params, pluralizationNumber: number): Promise<string> => {
        if (!msg) {
          return msg;
        }

        if (category === 'frontend') {
          throw new Error(`Category 'frontend' is reserved for frontend messages, use any other category for backend messages`);
        }
        // console.log('🪲tr', msg, category, lang);

      // if lang is not supported, fallback to primaryLanguage, then to english
      if (!this.options.supportedLanguages.includes(lang as SupportedLanguage)) {
        lang = this.primaryLanguage; // fallback to primary language first
        if (!this.options.supportedLanguages.includes(lang as SupportedLanguage)) {
          lang = 'en'; // final fallback to english
        }
      }

        let result;
        // try to get translation from cache
        const cacheKey = `${resourceConfig.resourceId}:${category}:${lang}:${msg}`;
        const cached = await this.cache.get(cacheKey);
        if (cached) {
          result = cached;
        }
        if (!result) {
          const resource = adminforth.resource(resourceConfig.resourceId);
          const translation = await resource.get([Filters.EQ(this.enFieldName, msg), Filters.EQ(this.options.categoryFieldName, category)]);
          if (!translation) {
            await resource.create({
              [this.enFieldName]: msg,
              [this.options.categoryFieldName]: category,
            });
            this.updateUntranslatedMenuBadge();
          }

          // do this check here, to faster register missing translations
          // also not cache it - no sense to cache english strings
          if (lang === 'en') {
            // set to cache to return faster next time
            result = msg;
          } else {
            result = translation?.[this.trFieldNames[lang]];
            if (!result) {
              // return english
              result = msg;
            }
          }
          // cache so even if key does not exist, we will not hit database
          await this.cache.set(cacheKey, result);
        }
        // if msg has '|' in it, then we need to aplly pluralization
        if (msg.includes('|')) {
          result = this.applyPluralization(result, pluralizationNumber, lang);
        }

        if (params) {
          for (const [key, value] of Object.entries(params)) {
            if (result) {
              result = result.replace(`{${key}}`, value);
            }
          }
        }
        return result;
      }
    }
  }

  applyPluralization(str: string, number: number, lang: string): string {
    const choices = str.split('|');
    const choicesLength = choices.length;

    if (choicesLength > 2) {
      // this is slavic pluralization
      return choices[this.slavicPluralRule(number, choicesLength)];
    } else {
      return number === 1 ? choices[0] : choices[1];
    }
  }
  
    // taken from here https://vue-i18n.intlify.dev/guide/essentials/pluralization.html#custom-pluralization
  slavicPluralRule(choice, choicesLength) {
    if (choice === 0) {
      return 0
    }

    const teen = choice > 10 && choice < 20
    const endsWithOne = choice % 10 === 1

    if (!teen && endsWithOne) {
      return 1
    }
    if (!teen && choice % 10 >= 2 && choice % 10 <= 4) {
      return 2
    }

    return choicesLength < 4 ? 2 : 3
  }
  instanceUniqueRepresentation(pluginOptions: any) : string {
    // optional method to return unique string representation of plugin instance. 
    // Needed if plugin can have multiple instances on one resource 
    return `single`;
  }

  async getCategoryTranslations(category: string, lang: string): Promise<Record<string, string>> {
    const resource = this.adminforth.resource(this.resourceConfig.resourceId);
    const cacheKey = `${this.resourceConfig.resourceId}:${category}:${lang}`;
    const cached = await this.cache.get(cacheKey);
    if (cached) {
      return cached;
    }
    const translations = {};
    const allTranslations = await resource.list([Filters.EQ(this.options.categoryFieldName, category)]);
    for (const tr of allTranslations) {
      const translatedValue = tr[this.trFieldNames[lang]];
      translations[tr[this.enFieldName]] = translatedValue || tr[this.enFieldName];
    }
    await this.cache.set(cacheKey, translations);
    return translations;
  }

  async languagesList(): Promise<{
    code: SupportedLanguage;
    nameOnNative: string;
    nameEnglish: string;
    emojiFlag: string;
    svgFlagB64: string;
  }[]> {
    return Promise.all(this.options.supportedLanguages.map(async (lang) => {
      const flagEmoji =
        typeof getFlagEmoji === 'function'
          ? getFlagEmoji
          : getFlagEmoji.default;
      

      return {
        code: lang,
        nameOnNative: iso6391.getNativeName(getPrimaryLanguageCode(lang)),
        nameEnglish: iso6391.getName(getPrimaryLanguageCode(lang)),
        emojiFlag: getCountryCodeFromLangCode(lang).toUpperCase().replace(/./g, char => String.fromCodePoint(char.charCodeAt(0) + 127397)),
        svgFlagB64: await flagEmoji(getCountryCodeFromLangCode(lang)),
      };
    }));
  }

  async feedCategoryTranslations(messages: {
    en_string: string;
    source: string;
  }[], category: string): Promise<void> {
    const adminforth = this.adminforth;
    const missingKeysDeduplicated = messages.reduce((acc: any[], missingKey: any) => {
      if (!acc.find((a) => a.en_string === missingKey.en_string)) {
        acc.push(missingKey);
      }
      return acc;
    }, []);

    await Promise.all(missingKeysDeduplicated.map(async (missingKey: any) => {
      const key = missingKey.en_string;
      const source = missingKey.source;
      const exists = await adminforth.resource(this.resourceConfig.resourceId).count([
        Filters.EQ(this.enFieldName, key), Filters.EQ(this.options.categoryFieldName, category)
      ]);
      if (exists) {
        return;
      }
      if (!key) {
        console.error(`Faced an empty key in feeding ${category} messages, source ${source}`);
      }
      const record = {
        [this.enFieldName]: key,
        [this.options.categoryFieldName]: category,
        ...(this.options.sourceFieldName ? { [this.options.sourceFieldName]: source } : {}),
      };
      try {
        await adminforth.resource(this.resourceConfig.resourceId).create(record);
      } catch (e) {
        console.error('🐛 Error creating record', e);
      }
    }));

    // updateBadge
    this.updateUntranslatedMenuBadge();
  }


  setupEndpoints(server: IHttpServer) {
    server.endpoint({
      method: 'GET',
      path: `/plugin/${this.pluginInstanceId}/frontend_messages`,
      noAuth: true,
      handler: async ({ query }) => {
        const lang = query.lang;
        
        // form map of translations
        const translations = await this.getCategoryTranslations('frontend', lang);
        return translations;

      }
    });

    const lock = new RAMLock();

    server.endpoint({
      method: 'POST',
      path: `/plugin/${this.pluginInstanceId}/update-field`,
      handler: async ({ body, adminUser, headers }) => {
        const { resourceId, recordId, field, value, reviewed } = body;

        const resource = this.adminforth.config.resources.find(r => r.resourceId === resourceId);
        // Create update object with just the single field
        const updateRecord = { [field]: value };
        
        // Use AdminForth's built-in update method
        const connector = this.adminforth.connectors[resource.dataSource];

        let oldRecord;
        let result;
        await lock.run(`edit-trans-${recordId}`,  async () => {
          // put into lock so 2 editors will not update the same record at the same time
          oldRecord = await connector.getRecordByPrimaryKey(resource, recordId)

          if (this.options.reviewedCheckboxesFieldName) {
            let oldValue;
            if (!oldRecord[this.options.reviewedCheckboxesFieldName]) {
              oldValue = {}
            } else {
              oldValue = {...oldRecord[this.options.reviewedCheckboxesFieldName]};
            }
            oldValue[field] = reviewed;
            updateRecord[this.options.reviewedCheckboxesFieldName] = { ...oldValue };
          }

          result = await this.adminforth.updateResourceRecord({
            resource,
            recordId,
            record: updateRecord,
            oldRecord,
            adminUser
          });
        });

        if (result.error) {
          return { error: result.error };
        }

        const updatedRecord = await connector.getRecordByPrimaryKey(resource, recordId);

        return { record: updatedRecord };
      }
    });

    server.endpoint({
      method: 'POST',
      path: `/plugin/${this.pluginInstanceId}/translate-selected-to-languages`,
      noAuth: false,
      handler: async ({ body, tr, adminUser }) => {
        const selectedLanguages = body.selectedLanguages;
        const selectedIds = body.selectedIds;
        
        const jobId = await this.bulkTranslate({ selectedIds, selectedLanguages, adminUser });

        return { 
          ok: true, 
          jobId: jobId,
        };
      }
    });

    server.endpoint({
      method: 'POST',
      path: `/plugin/${this.pluginInstanceId}/get_filtered_ids`,
      handler: async ({ body, adminUser, headers, query, cookies, requestUrl }) => {
        const resource = this.resourceConfig;

        for (const hook of resource.hooks?.list?.beforeDatasourceRequest || []) {
          const filterTools = filtersTools.get(body);
          body.filtersTools = filterTools;
          const resp = await hook({
            resource,
            query: body,
            adminUser,
            //@ts-ignore
            filtersTools: filterTools, 
            extra: {
              body, query, headers, cookies, requestUrl
            },
            adminforth: this.adminforth,
          });
          if (!resp || (!resp.ok && !resp.error)) {
            throw new Error(`Hook must return object with {ok: true} or { error: 'Error' } `);
          }
          if (resp.error) {
            return { error: resp.error };
          }
        }
        const filters = body.filters;

        const normalizedFilters = { operator: AdminForthFilterOperators.AND, subFilters: [] };
        if (filters) {
          if (typeof filters !== 'object') {
            throw new Error(`Filter should be an array or an object`);
          }
          if (Array.isArray(filters)) {
            // if filters are an array, they will be connected with "AND" operator by default
            normalizedFilters.subFilters = filters;
          } else if (filters.field) {
            // assume filter is a SingleFilter
            normalizedFilters.subFilters = [filters];
          } else if (filters.subFilters) {
            // assume filter is a AndOr filter
            normalizedFilters.operator = filters.operator;
            normalizedFilters.subFilters = filters.subFilters;
          } else {
            // wrong filter
            throw new Error(`Wrong filter object value: ${JSON.stringify(filters)}`);
          }
        }

        const records = await this.adminforth.resource(this.resourceConfig.resourceId).list(normalizedFilters);
        if (!records) {
          return { ok: true, recordIds: [] };
        }
        const primaryKeyColumn = this.resourceConfig.columns.find((col) => col.primaryKey);

        const recordIds = records.map(record => record[primaryKeyColumn.name]);

        return { ok: true, recordIds }
      }
    });

  }

}