// The register of food sources (20 SRC table, FD-02a, FD-49, FD-73): one entry per source file the build imports, with its
// importer, the release in hand, how often it is refreshed and where a new release shows. food-build-import reads the
// importer lines from here; food-update (scripts/food-update.mjs) reads the cadence and runs the release checks.
//
// cadence: `weekly` — a new export is downloaded and imported on every run (Open Food Facts); `agency` — imported when the
// agency publishes: USDA files are downloaded by the weekly run itself, other tables once a session has put the new file
// here and moved `file`, `released` and `seen` (the check names the release every week until then); `static` — no further
// releases (SR Legacy, TACO 4th edition, Oxford 2024). USDA Branded comes out with the other FDC files, not monthly.
// check: a page whose release markers change when the agency publishes; `seen` is the fingerprint of the markers that
// matched the imported release (releaseFingerprint below), so a different fingerprint is a release not imported yet.

export type SourceCadence = 'weekly' | 'agency' | 'static';
export type SourceCheck = { page: string; pattern: RegExp; seen: string | null };
export type FoodSource = {
  name: string; src: `SRC-${string}`; market: string; cadence: SourceCadence;
  // The importer, the file or folder under the food data folder, and the publication date of that file (the release date).
  importer: 'usda' | 'national' | 'off' | 'oxford'; dataset?: string; file: string; released: string;
  check?: SourceCheck;
  // Why a source has no automatic check.
  unchecked?: string;
};

const usdaPage = 'https://fdc.nal.usda.gov/download-datasets';
export const foodSources: Record<string, FoodSource> = {
  foundation: { name: 'USDA Foundation', src: 'SRC-01', market: 'US', cadence: 'agency', importer: 'usda', dataset: 'foundation',
    file: 'usda/x-FoodData_Central_foundation_food_json_2026-04-30/FoodData_Central_foundation_food_json_2026-04-30.json', released: '2026-04-30',
    check: { page: usdaPage, pattern: /FoodData_Central_foundation_food_json_[\d-]+\.zip/g, seen: 'df8e69472bf98efb' } },
  sr_legacy: { name: 'USDA SR Legacy', src: 'SRC-01', market: 'US', cadence: 'static', importer: 'usda', dataset: 'sr_legacy',
    file: 'usda/x-FoodData_Central_sr_legacy_food_json_2018-04/FoodData_Central_sr_legacy_food_json_2018-04.json', released: '2018-04-01',
    unchecked: 'the final release of SR Legacy (2018)' },
  survey: { name: 'USDA FNDDS', src: 'SRC-01', market: 'US', cadence: 'agency', importer: 'usda', dataset: 'survey',
    file: 'usda/x-FoodData_Central_survey_food_json_2024-10-31/surveyDownload.json', released: '2024-10-31',
    check: { page: usdaPage, pattern: /FoodData_Central_survey_food_json_[\d-]+\.zip/g, seen: '3efc931e685168f7' } },
  cofid: { name: 'CoFID', src: 'SRC-03', market: 'GB', cadence: 'agency', importer: 'national',
    file: 'cofid/McCance_Widdowsons_Composition_of_Foods_Integrated_Dataset_2021.xlsx', released: '2021-03-19',
    check: { page: 'https://www.gov.uk/government/publications/composition-of-foods-integrated-dataset-cofid', pattern: /[\w%.-]+\.xlsx/g, seen: 'fdd0c5306e1a9589' } },
  cnf: { name: 'Canadian Nutrient File', src: 'SRC-04', market: 'CA', cadence: 'agency', importer: 'national',
    file: 'cnf/cnf_fcen_all-files-data_2026.zip', released: '2026-05-26',
    check: { page: 'https://open.canada.ca/data/api/action/package_search?q=%22canadian%20nutrient%20file%22&rows=10', pattern: /Canadian Nutrient File, \d{4}/g, seen: '6eaa10e13791155f' } },
  afcd: { name: 'AFCD', src: 'SRC-05', market: 'AU', cadence: 'agency', importer: 'national', file: 'afcd', released: '2025-12-23',
    check: { page: 'https://www.foodstandards.gov.au/science-data/food-nutrient-databases/afcd/data-files', pattern: /[\w%.-]+\.xlsx(?:\?v=\d+)?/g, seen: '91df39e8cbf70150' } },
  ausnut: { name: 'AUSNUT', src: 'SRC-09', market: 'AU', cadence: 'agency', importer: 'national', file: 'ausnut', released: '2026-03-25',
    check: { page: 'https://www.foodstandards.gov.au/science-data/food-nutrient-databases/ausnut/food-nutrients', pattern: /\/sites\/default\/files\/[^"?\s]+\.xlsx(?:\?v=\d+)?/g, seen: 'beccfd60c09f9f47' } },
  ciqual: { name: 'Ciqual', src: 'SRC-10', market: 'FR', cadence: 'agency', importer: 'national', file: 'ciqual', released: '2025-11-03',
    check: { page: 'https://www.data.gouv.fr/api/1/datasets/?q=ciqual&page_size=20', pattern: /Ciqual 20\d\d/g, seen: 'd723bdaa8ce56364' } },
  bls: { name: 'BLS', src: 'SRC-15', market: 'DE', cadence: 'agency', importer: 'national', file: 'bls/BLS_4_0_2025_DE.zip', released: '2025-12-17',
    check: { page: 'https://www.blsdb.de/download', pattern: /doi\.org\/10\.25826\/Data[\w-]+|BLS_[\w.]+?\.zip/g, seen: '73975569267e87ba' } },
  frida: { name: 'Frida', src: 'SRC-14', market: 'DK', cadence: 'agency', importer: 'national', file: 'frida/FCDB_6.1_Dataset.xlsx', released: '2026-06-09',
    check: { page: 'https://api.figshare.com/v2/articles/32312844', pattern: /"title":\s*"The Danish Food Composition Database, version [\d.]+"/g, seen: 'd39c3ced968fa1db' } },
  livsmedelsverket: { name: 'Livsmedelsdatabasen', src: 'SRC-12', market: 'SE', cadence: 'agency', importer: 'national',
    file: 'livsmedelsverket/livsmedel-en.json', released: '2026-07-01',
    check: { page: 'https://dataportal.livsmedelsverket.se/livsmedel/api/v1/livsmedel?offset=0&limit=1&sprak=2', pattern: /"version":\s*"[^"]+"/g, seen: 'ef864f40486b93b3' } },
  matvaretabellen: { name: 'Matvaretabellen', src: 'SRC-13', market: 'NO', cadence: 'agency', importer: 'national', file: 'matvaretabellen', released: '2026-01-01',
    check: { page: 'https://www.matvaretabellen.no/en/', pattern: /Matvaretabellen 20\d\d/g, seen: '5d872c16fd684045' } },
  fineli: { name: 'Fineli', src: 'SRC-16', market: 'FI', cadence: 'agency', importer: 'national', file: 'fineli/src/basic-package-1', released: '2017-04-30',
    unchecked: 'fineli.fi answers every script with a bot check (403); the imported package is THL release 18 from a public mirror' },
  tfda: { name: 'TFDA', src: 'SRC-20', market: 'TW', cadence: 'agency', importer: 'national', file: 'tfda/tfda-export-20.csv.zip', released: '2026-10-05',
    check: { page: 'https://data.gov.tw/api/v2/rest/dataset/8543', pattern: /"modifiedDate":\s*"[^"]+"/g, seen: 'ee4ee6d8b98de868' } },
  blv: { name: 'BLV', src: 'SRC-21', market: 'CH', cadence: 'agency', importer: 'national', file: 'blv/Schweizer_Nahrwertdatenbank.xlsx', released: '2026-07-13',
    check: { page: 'https://naehrwertdaten.ch/de/downloads/', pattern: /Version [\d.]+ der Schweizer N\S+hrwertdatenbank/g, seen: '09e6a89b6b5916cd' } },
  mext: { name: 'MEXT', src: 'SRC-22', market: 'JP', cadence: 'agency', importer: 'national', file: 'mext/mext_00001_chapter2_data.xlsx', released: '2023-01-01',
    check: { page: 'https://www.mext.go.jp/a_menu/syokuhinseibun/mext_00001.html', pattern: /[\w%.-]+\.xlsx/g, seen: 'd39ff7e9b9de2de9' } },
  taco: { name: 'TACO', src: 'SRC-23', market: 'BR', cadence: 'static', importer: 'national', file: 'taco/taco_4_edicao_ampliada_e_revisada.pdf', released: '2011-01-01',
    unchecked: 'the 4th edition (2011) is the last; NEPA-UNICAMP publishes no data files' },
  oxford: { name: 'Oxford chain menus 2024', src: 'SRC-07', market: 'GB', cadence: 'static', importer: 'oxford', file: 'chains/oxford-2024/Dataset.xlsx', released: '2024-01-01',
    unchecked: 'a fixed research dataset; chain menus are refreshed by FD-28' },
  branded: { name: 'USDA Branded', src: 'SRC-02', market: 'US', cadence: 'agency', importer: 'usda', dataset: 'branded',
    file: 'usda/x-FoodData_Central_branded_food_json_2026-04-30/FoodData_Central_branded_food_json_2026-04-30.json', released: '2026-04-30',
    check: { page: usdaPage, pattern: /FoodData_Central_branded_food_json_[\d-]+\.zip/g, seen: '8d8da62cbfce94c3' } },
  off: { name: 'Open Food Facts', src: 'SRC-06', market: '*', cadence: 'weekly', importer: 'off', file: 'off/openfoodfacts-products.jsonl.gz', released: '2026-10-12',
    unchecked: 'a new export every day; the weekly run downloads the newest' },
};

// The export Open Food Facts publishes every day; a weekly run takes the newest.
export const OFF_EXPORT_URL = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz';
// The countries the Open Food Facts importer reads.
export const OFF_COUNTRIES = 'US,GB,CA,AU,NZ,DE,FR,ES,IT,NL,BE,AT,CH,PT,PL,SE,NO,DK,FI,IE';
// USDA FoodData Central data files: <dataset>_food_json_<date>.zip.
export const usdaZipUrl = (dataset: string, date: string) => `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_${dataset}_food_json_${date}.zip`;

// The release markers of a page, entity-decoded, deduplicated and sorted, and their short hash.
export function releaseMarkers(page: string, pattern: RegExp) {
  const text = page.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));
  return [...new Set(text.match(new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`)) ?? [])].sort();
}
export async function releaseFingerprint(markers: string[]) {
  const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(markers.join('\n')));
  return Buffer.from(digest).toString('hex').slice(0, 16);
}

// Reads one source's release page: the markers, their fingerprint and whether it differs from the imported release.
// No markers is a failed check (the page changed shape or refused), not an unchanged one.
export async function checkSource(source: FoodSource, fetcher: typeof fetch = fetch) {
  if (!source.check) return { status: 'unchecked' as const, why: source.unchecked ?? 'no release page' };
  try {
    const response = await fetcher(source.check.page, { headers: { 'user-agent': 'Calsa food source check (https://calsa.app)' },
      signal: AbortSignal.timeout(60_000), redirect: 'follow' });
    if (!response.ok) return { status: 'failed' as const, why: `the page answered ${response.status}` };
    const markers = releaseMarkers(await response.text(), source.check.pattern);
    if (!markers.length) return { status: 'failed' as const, why: 'no release markers on the page' };
    const fingerprint = await releaseFingerprint(markers);
    return { status: fingerprint === source.check.seen ? 'current' as const : 'new' as const, fingerprint, markers };
  } catch (error) { return { status: 'failed' as const, why: (error as Error).message }; }
}