feat!: optional fail-closed fallback, legend output, richer prompt & docs
BREAKING CHANGE: regex fallback is now opt-in (patterns no longer defaults
to presets.swiss). With no fallback an LLM failure throws AnonymizationError
(fail-closed) and the pre-filter is bypassed; at least one of llm/patterns is
required. AnonymizationResult gains a required `legend`; anonymizeChunks seed
is now { mapping, legend? } and returns legend.
- prompt: model may coin new UPPERCASE abbreviations and returns a 'legende'
explaining every abbreviation used (French); backfilled by DEFAULT_LEGEND
- PatternDef.meaning surfaces in the legend; swiss/generic presets get meanings
- AnonymizationError (exported) wraps the cause on fail-closed
- README: drop the chatbot provenance line; add 'How it works' + nLPD sections
- 34 tests / 99% coverage; bump to 0.2.0
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { RegexFallback } from './fallback.js';
|
||||
import { presets } from './presets.js';
|
||||
import { AnonymizationError } from './errors.js';
|
||||
import {
|
||||
type AnonymizationResult,
|
||||
type AnonymizerConfig,
|
||||
@@ -14,6 +14,36 @@ const NOOP_LOGGER: Logger = { warn: () => {} };
|
||||
// Likely proper-name heuristic (two capitalized words), in addition to the structured fallback.
|
||||
const DEFAULT_NAME_HINT = /\b[A-ZÀ-Ý][a-zà-ÿ]+\s+[A-ZÀ-Ý][a-zà-ÿ]+\b/;
|
||||
|
||||
/**
|
||||
* Built-in French meanings for the standard abbreviations, used to backfill the
|
||||
* legend when the model (or the regex fallback) doesn't supply one for an
|
||||
* abbreviation that actually appears in the output.
|
||||
*/
|
||||
const DEFAULT_LEGEND: Record<string, string> = {
|
||||
PER: 'Personne',
|
||||
ORG: 'Organisation',
|
||||
LOC: 'Lieu',
|
||||
NOM: 'Nom de famille',
|
||||
PRENOM: 'Prénom',
|
||||
DATE_NAISSANCE: 'Date de naissance',
|
||||
AGE: 'Âge',
|
||||
ADRESSE: 'Adresse',
|
||||
EMAIL: 'Adresse e-mail',
|
||||
TELEPHONE: 'Numéro de téléphone',
|
||||
TEL: 'Numéro de téléphone',
|
||||
AVS: 'Numéro AVS',
|
||||
IBAN: 'IBAN',
|
||||
NSS: 'Numéro de sécurité sociale',
|
||||
DATE: 'Date',
|
||||
IPV4: 'Adresse IP',
|
||||
CREDIT_CARD: 'Carte de crédit',
|
||||
M: 'Masculin',
|
||||
F: 'Féminin',
|
||||
U: 'Inconnu',
|
||||
Mineur: 'Mineur',
|
||||
Adulte: 'Adulte',
|
||||
};
|
||||
|
||||
/**
|
||||
* Return a stateless copy of a regex: a global/sticky (`g`/`y`) regex carries a
|
||||
* mutable `lastIndex`, which makes repeated `.test()` calls return alternating
|
||||
@@ -27,24 +57,31 @@ function stateless(re: RegExp): RegExp {
|
||||
* Framework-agnostic anonymization engine.
|
||||
*
|
||||
* `anonymize()`: pre-filter (skips the LLM when there is no PII) → LLM →
|
||||
* regex fallback on failure/timeout → bidirectional validation.
|
||||
* regex fallback on failure/timeout → bidirectional validation. When no fallback
|
||||
* is configured, an LLM failure throws {@link AnonymizationError} (fail-closed).
|
||||
* `deanonymize()` restores the real values. `makeStreamDeanonymizer()` handles
|
||||
* the critical case of a placeholder fragmented across several streamed tokens.
|
||||
*
|
||||
* Pass an {@link LlmProvider} to detect free-form PII (names, addresses);
|
||||
* omit it for deterministic regex-only mode.
|
||||
* Each result carries a `legend` (abbreviation → meaning) you can forward to a
|
||||
* downstream LLM so it understands the placeholder tokens.
|
||||
*
|
||||
* Pass an {@link LlmProvider} to detect free-form PII (names, addresses) and/or
|
||||
* `patterns` for the regex fallback. At least one of the two is required.
|
||||
*/
|
||||
export class Anonymizer {
|
||||
private readonly llm?: LlmProvider;
|
||||
private readonly fallback: RegexFallback;
|
||||
private readonly fallback?: RegexFallback;
|
||||
private readonly nameHint: RegExp;
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(config: AnonymizerConfig = {}) {
|
||||
this.llm = config.llm;
|
||||
this.fallback = new RegexFallback(config.patterns ?? presets.swiss);
|
||||
this.fallback = config.patterns ? new RegexFallback(config.patterns) : undefined;
|
||||
this.nameHint = stateless(config.nameHint ?? DEFAULT_NAME_HINT);
|
||||
this.logger = config.logger ?? NOOP_LOGGER;
|
||||
if (!this.llm && !this.fallback) {
|
||||
throw new AnonymizationError('Anonymizer requires an LlmProvider, regex patterns, or both');
|
||||
}
|
||||
}
|
||||
|
||||
private hasLlm(): boolean {
|
||||
@@ -52,51 +89,67 @@ export class Anonymizer {
|
||||
}
|
||||
|
||||
private likelyHasPii(text: string): boolean {
|
||||
return this.fallback.hasPii(text) || this.nameHint.test(text);
|
||||
return (this.fallback?.hasPii(text) ?? false) || this.nameHint.test(text);
|
||||
}
|
||||
|
||||
async anonymize(text: string): Promise<AnonymizationResult> {
|
||||
// Pre-filter: most text has no PII → avoid the LLM round-trip (latency + cost)
|
||||
// and return the text unchanged.
|
||||
if (!this.likelyHasPii(text)) return { anon: text, mapping: {} };
|
||||
// Pre-filter: most text has no PII → avoid the LLM round-trip. Only safe when
|
||||
// a regex fallback exists to judge "no PII"; without one we always consult the
|
||||
// LLM so structured PII (e.g. emails) can't slip through unanonymized.
|
||||
if (this.fallback && !this.likelyHasPii(text)) return { anon: text, mapping: {}, legend: {} };
|
||||
|
||||
if (this.hasLlm()) {
|
||||
try {
|
||||
const result = await this.llm!.anonymize(text);
|
||||
this.validate(result); // throws if inconsistent → fall back
|
||||
return result;
|
||||
return {
|
||||
anon: result.anon,
|
||||
mapping: result.mapping,
|
||||
legend: this.buildLegend(result.anon, result.legend),
|
||||
};
|
||||
} catch (err) {
|
||||
if (!this.fallback) {
|
||||
throw new AnonymizationError(`anonymize failed: ${(err as Error).message}`, { cause: err });
|
||||
}
|
||||
this.logger.warn(`LLM unavailable/invalid → regex fallback: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
return this.fallback.anonymize(text);
|
||||
return this.fallbackAnonymize(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Anonymize retrieved chunks BEFORE sending them on, CONSISTENTLY with the
|
||||
* question's mapping (`seed`). Returns the anonymized chunks + the COMPLETE
|
||||
* mapping (question ∪ chunks). Strategy:
|
||||
* question's `seed` ({ mapping, legend }). Returns the anonymized chunks + the
|
||||
* COMPLETE mapping and legend (question ∪ chunks). Strategy:
|
||||
* 1. Deterministic reuse: replace any already-known value (from the question)
|
||||
* by its placeholder → question↔chunk coreference with no LLM call.
|
||||
* 2. Pre-filter: if no residual PII remains → return as-is (most corpus chunks
|
||||
* have no PII → LLM cost avoided).
|
||||
* have no PII → LLM cost avoided). Skipped when no regex fallback is set.
|
||||
* 3. Otherwise: ONE batched LLM call, then de-collision merge (renumber the
|
||||
* genuinely new entities). Failure/timeout → regex fallback per chunk.
|
||||
* genuinely new entities). Failure/timeout → regex fallback per chunk, or
|
||||
* {@link AnonymizationError} when no fallback is configured.
|
||||
*/
|
||||
async anonymizeChunks(
|
||||
chunks: string[],
|
||||
seed: Record<string, string>,
|
||||
): Promise<{ anon: string[]; mapping: Record<string, string> }> {
|
||||
if (!chunks.length) return { anon: [], mapping: { ...seed } };
|
||||
seed: { mapping: Record<string, string>; legend?: Record<string, string> },
|
||||
): Promise<{ anon: string[]; mapping: Record<string, string>; legend: Record<string, string> }> {
|
||||
const seedLegend = seed.legend ?? {};
|
||||
if (!chunks.length) return { anon: [], mapping: { ...seed.mapping }, legend: { ...seedLegend } };
|
||||
|
||||
const seeded = chunks.map((c) => this.applyKnown(c, seed));
|
||||
if (!seeded.some((c) => this.likelyHasPii(c))) return { anon: seeded, mapping: { ...seed } };
|
||||
const seeded = chunks.map((c) => this.applyKnown(c, seed.mapping));
|
||||
if (this.fallback && !seeded.some((c) => this.likelyHasPii(c))) {
|
||||
return { anon: seeded, mapping: { ...seed.mapping }, legend: { ...seedLegend } };
|
||||
}
|
||||
|
||||
if (this.hasLlm()) {
|
||||
try {
|
||||
const { segments, mapping: gMap } = await this.llm!.anonymizeBatch(seeded, Object.keys(seed));
|
||||
const {
|
||||
segments,
|
||||
mapping: gMap,
|
||||
legend: gLegend,
|
||||
} = await this.llm!.anonymizeBatch(seeded, Object.keys(seed.mapping));
|
||||
if (segments.length !== seeded.length) throw new Error('SEGMENT_COUNT_MISMATCH');
|
||||
const { mapping, rename } = this.mergeMappings(seed, gMap);
|
||||
const { mapping, rename } = this.mergeMappings(seed.mapping, gMap);
|
||||
const anon = segments.map((s) => this.applyRename(s, rename));
|
||||
// Anti-leak: every placeholder present in a chunk must be mapped.
|
||||
for (const s of anon) {
|
||||
@@ -104,12 +157,16 @@ export class Anonymizer {
|
||||
if (!(ph in mapping)) throw new Error(`unmapped placeholder ${ph}`);
|
||||
}
|
||||
}
|
||||
return { anon, mapping };
|
||||
const legend = this.buildLegend(anon.join('\n'), { ...seedLegend, ...gLegend });
|
||||
return { anon, mapping, legend };
|
||||
} catch (err) {
|
||||
if (!this.fallback) {
|
||||
throw new AnonymizationError(`anonymizeChunks failed: ${(err as Error).message}`, { cause: err });
|
||||
}
|
||||
this.logger.warn(`Chunk anonymization → regex fallback: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
return this.fallbackChunks(seeded, seed);
|
||||
return this.fallbackChunks(seeded, seed.mapping, seedLegend);
|
||||
}
|
||||
|
||||
/** Replace known values (seed) by their placeholder, longest values first. */
|
||||
@@ -135,6 +192,32 @@ export class Anonymizer {
|
||||
return m ? { type: m[1], index: Number(m[2]), suffix: m[3] ?? '' } : null;
|
||||
}
|
||||
|
||||
/** Abbreviations used by a placeholder: entity type, attribute, and all-caps context codes. */
|
||||
private usedAbbreviations(text: string): Set<string> {
|
||||
const out = new Set<string>();
|
||||
for (const ph of text.match(PLACEHOLDER_RE) ?? []) {
|
||||
const m = /^\[([A-Z]+)_\d+(?:\.([A-Z_]+):([^\]]+))?\]$/.exec(ph);
|
||||
if (!m) continue;
|
||||
out.add(m[1]); // entity type
|
||||
if (m[2]) out.add(m[2]); // attribute
|
||||
if (m[3] && /^[A-Z]+$/.test(m[3])) out.add(m[3]); // context, only if it's a code (not a value)
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a legend covering every abbreviation that appears in `text`, taking the
|
||||
* meaning from `provided` (LLM/fallback) → {@link DEFAULT_LEGEND} → the
|
||||
* abbreviation itself. Guarantees coverage even if the model omits entries.
|
||||
*/
|
||||
private buildLegend(text: string, provided: Record<string, string>): Record<string, string> {
|
||||
const legend: Record<string, string> = {};
|
||||
for (const ab of this.usedAbbreviations(text)) {
|
||||
legend[ab] = provided[ab] ?? DEFAULT_LEGEND[ab] ?? ab;
|
||||
}
|
||||
return legend;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge the chunks' LLM mapping into the question's (seed).
|
||||
* - value already known → reuse the question's placeholder (rename).
|
||||
@@ -181,20 +264,29 @@ export class Anonymizer {
|
||||
return { mapping, rename };
|
||||
}
|
||||
|
||||
/** Single-text regex fallback, routed through the legend builder. */
|
||||
private fallbackAnonymize(text: string): AnonymizationResult {
|
||||
const r = this.fallback!.anonymize(text);
|
||||
return { anon: r.anon, mapping: r.mapping, legend: this.buildLegend(r.anon, r.legend) };
|
||||
}
|
||||
|
||||
/** Fallback (no LLM): regex per chunk, de-collision merged into the seed. */
|
||||
private fallbackChunks(
|
||||
seeded: string[],
|
||||
seed: Record<string, string>,
|
||||
): { anon: string[]; mapping: Record<string, string> } {
|
||||
seedLegend: Record<string, string>,
|
||||
): { anon: string[]; mapping: Record<string, string>; legend: Record<string, string> } {
|
||||
let mapping: Record<string, string> = { ...seed };
|
||||
let provided: Record<string, string> = { ...seedLegend };
|
||||
const anon: string[] = [];
|
||||
for (const c of seeded) {
|
||||
const r = this.fallback.anonymize(c);
|
||||
const r = this.fallback!.anonymize(c);
|
||||
const { mapping: merged, rename } = this.mergeMappings(mapping, r.mapping);
|
||||
mapping = merged;
|
||||
provided = { ...provided, ...r.legend };
|
||||
anon.push(this.applyRename(r.anon, rename));
|
||||
}
|
||||
return { anon, mapping };
|
||||
return { anon, mapping, legend: this.buildLegend(anon.join('\n'), provided) };
|
||||
}
|
||||
|
||||
/** Every placeholder in the text must be in the mapping AND vice-versa (anti-leak). */
|
||||
|
||||
11
src/errors.ts
Normal file
11
src/errors.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Thrown when anonymization cannot be completed and there is no fallback to
|
||||
* degrade to — a fail-closed signal so the caller never proceeds with
|
||||
* un-anonymized text. The originating error is preserved in `.cause`.
|
||||
*/
|
||||
export class AnonymizationError extends Error {
|
||||
constructor(message: string, options?: { cause?: unknown }) {
|
||||
super(message, options);
|
||||
this.name = 'AnonymizationError';
|
||||
}
|
||||
}
|
||||
@@ -13,10 +13,11 @@ export class RegexFallback {
|
||||
|
||||
anonymize(text: string): AnonymizationResult {
|
||||
const mapping: Record<string, string> = {};
|
||||
const legend: Record<string, string> = {};
|
||||
const counters: Record<string, number> = {};
|
||||
let anon = text;
|
||||
|
||||
for (const { tag, re, validate } of this.patterns) {
|
||||
for (const { tag, re, validate, meaning } of this.patterns) {
|
||||
anon = anon.replace(re, (match) => {
|
||||
// A validator can reject a regex match (e.g. Luhn) → leave the text as-is.
|
||||
if (validate && !validate(match)) return match;
|
||||
@@ -26,10 +27,11 @@ export class RegexFallback {
|
||||
counters[tag] = (counters[tag] ?? 0) + 1;
|
||||
const ph = `[${tag}_${counters[tag]}]`;
|
||||
mapping[ph] = match;
|
||||
legend[tag] = meaning ?? tag; // document the abbreviation used
|
||||
return ph;
|
||||
});
|
||||
}
|
||||
return { anon, mapping };
|
||||
return { anon, mapping, legend };
|
||||
}
|
||||
|
||||
/** Fast pre-check: is there any structured identifier worth anonymizing? */
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
export { Anonymizer } from './anonymizer.js';
|
||||
export { AnonymizationError } from './errors.js';
|
||||
export { RegexFallback } from './fallback.js';
|
||||
export { presets } from './presets.js';
|
||||
export {
|
||||
|
||||
@@ -5,11 +5,11 @@ import type { PatternDef } from './types.js';
|
||||
* Order matters: specific patterns (AVS, IBAN) come before generic ones.
|
||||
*/
|
||||
const swiss: PatternDef[] = [
|
||||
{ tag: 'AVS', re: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g }, // Swiss social security (AVS/AHV)
|
||||
{ tag: 'IBAN', re: /\bCH\d{2}[0-9A-Z]{17}\b/gi }, // Swiss IBAN
|
||||
{ tag: 'EMAIL', re: /\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g },
|
||||
{ tag: 'TEL', re: /(?:\+41|0)(?:[\s.-]?\d){9}\b/g }, // Swiss phone number
|
||||
{ tag: 'DATE', re: /\b\d{1,2}[./-]\d{1,2}[./-]\d{2,4}\b/g },
|
||||
{ tag: 'AVS', re: /\b756\.\d{4}\.\d{4}\.\d{2}\b/g, meaning: 'Numéro AVS' }, // Swiss social security
|
||||
{ tag: 'IBAN', re: /\bCH\d{2}[0-9A-Z]{17}\b/gi, meaning: 'IBAN' }, // Swiss IBAN
|
||||
{ tag: 'EMAIL', re: /\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g, meaning: 'Adresse e-mail' },
|
||||
{ tag: 'TEL', re: /(?:\+41|0)(?:[\s.-]?\d){9}\b/g, meaning: 'Numéro de téléphone' }, // Swiss phone
|
||||
{ tag: 'DATE', re: /\b\d{1,2}[./-]\d{1,2}[./-]\d{2,4}\b/g, meaning: 'Date' },
|
||||
];
|
||||
|
||||
/** Luhn check on a credit-card candidate (ignores spaces/dashes); 13–19 digits. */
|
||||
@@ -39,16 +39,20 @@ function luhnValid(value: string): boolean {
|
||||
* claimed before phone numbers so they don't get mis-tagged as `TEL`.
|
||||
*/
|
||||
const generic: PatternDef[] = [
|
||||
{ tag: 'EMAIL', re: /\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g },
|
||||
{ tag: 'IBAN', re: /\b[A-Z]{2}\d{2}[0-9A-Z]{11,30}\b/g },
|
||||
{ tag: 'IPV4', re: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g },
|
||||
{ tag: 'EMAIL', re: /\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g, meaning: 'Adresse e-mail' },
|
||||
{ tag: 'IBAN', re: /\b[A-Z]{2}\d{2}[0-9A-Z]{11,30}\b/g, meaning: 'IBAN' },
|
||||
{ tag: 'IPV4', re: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g, meaning: 'Adresse IP' },
|
||||
// 13–19 digits in optional space/dash groups, gated by Luhn to avoid eating
|
||||
// arbitrary long digit runs (account numbers, ids).
|
||||
{ tag: 'CREDIT_CARD', re: /\b\d(?:[ -]?\d){12,18}\b/g, validate: luhnValid },
|
||||
{ tag: 'DATE', re: /\b\d{1,2}[./-]\d{1,2}[./-]\d{2,4}\b/g },
|
||||
{ tag: 'CREDIT_CARD', re: /\b\d(?:[ -]?\d){12,18}\b/g, validate: luhnValid, meaning: 'Carte de crédit' },
|
||||
{ tag: 'DATE', re: /\b\d{1,2}[./-]\d{1,2}[./-]\d{2,4}\b/g, meaning: 'Date' },
|
||||
// Requires grouped/separated digits (or a leading +country) so plain integers
|
||||
// aren't mistaken for phone numbers.
|
||||
{ tag: 'TEL', re: /(?:\+\d{1,3}[\s.-]?)?(?:\(?\d{2,4}\)?[\s.-]){2,3}\d{2,4}/g },
|
||||
{
|
||||
tag: 'TEL',
|
||||
re: /(?:\+\d{1,3}[\s.-]?)?(?:\(?\d{2,4}\)?[\s.-]){2,3}\d{2,4}/g,
|
||||
meaning: 'Numéro de téléphone',
|
||||
},
|
||||
];
|
||||
|
||||
/** Built-in pattern sets for the regex fallback. */
|
||||
|
||||
@@ -13,12 +13,24 @@ export const DEFAULT_SYSTEM_PROMPT = [
|
||||
' NOM:M|F|U · DATE_NAISSANCE:<année> · AGE:Mineur|Adulte',
|
||||
' ADRESSE:Lieu|Rue|Ville|NPA|Pays · ORG:Entreprise|Ecole',
|
||||
'',
|
||||
'ABRÉVIATIONS DYNAMIQUES:',
|
||||
'- Si tu rencontres un TYPE d’entité, un ATTRIBUT ou un CONTEXTE non listé ci-dessus,',
|
||||
' INVENTE une abréviation COURTE en MAJUSCULES (lettres A–Z et "_" uniquement, ex. PER, NOM, M),',
|
||||
' suivant la même convention, et réutilise-la de façon cohérente partout.',
|
||||
'- N’utilise JAMAIS d’espaces, accents, chiffres ou symboles dans une abréviation.',
|
||||
'',
|
||||
'LÉGENDE:',
|
||||
'- Renvoie une "legende" : un objet associant CHAQUE abréviation utilisée (entité, attribut, contexte)',
|
||||
' à sa signification complète en français. Ex.: {"PER":"Personne","NOM":"Nom de famille","M":"Masculin"}.',
|
||||
'- La légende sert à expliquer les placeholders à un autre modèle ; elle ne contient AUCUNE donnée réelle.',
|
||||
'',
|
||||
'RÈGLES:',
|
||||
'1. Coréférence: la MÊME personne garde le MÊME identifiant (PER_1) dans tout le texte.',
|
||||
'2. N’anonymise JAMAIS les termes pédagogiques/techniques (langages, concepts, titres de cours, fonctions).',
|
||||
'3. Si AUCUNE donnée personnelle: renvoie le texte original et "mapping": {}.',
|
||||
'3. Si AUCUNE donnée personnelle: renvoie le texte original, "mapping": {} et "legende": {}.',
|
||||
'4. N’anonymise pas un placeholder déjà présent (idempotence).',
|
||||
'5. Sortie STRICTEMENT JSON valide: {"texte_anonymise": "...", "mapping": {"[PER_1.NOM:M]": "..."}}.',
|
||||
'5. Sortie STRICTEMENT JSON valide:',
|
||||
' {"texte_anonymise": "...", "mapping": {"[PER_1.NOM:M]": "..."}, "legende": {"PER": "Personne", "NOM": "Nom de famille", "M": "Masculin"}}.',
|
||||
].join('\n');
|
||||
|
||||
export interface OpenAICompatibleOptions {
|
||||
@@ -48,6 +60,20 @@ const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout
|
||||
/** A transport error that the caller may retry (network / timeout / 429 / 5xx). */
|
||||
class TransientError extends Error {}
|
||||
|
||||
/**
|
||||
* Coerce the model's `legende` into a string→string map, lenient by design: a
|
||||
* missing or malformed legend yields `{}` rather than failing the anonymization
|
||||
* (the legend is metadata; the `mapping` is what guarantees no leak).
|
||||
*/
|
||||
function asLegend(raw: unknown): Record<string, string> {
|
||||
if (!raw || typeof raw !== 'object') return {};
|
||||
const out: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
|
||||
if (typeof v === 'string') out[k] = v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const DEFAULT_BATCH_INSTRUCTIONS = (usedIds: string[]): string =>
|
||||
'\n\nMODE LOT (segments) :' +
|
||||
'\n- ENTRÉE : un objet JSON {"segments": ["…", "…"]}.' +
|
||||
@@ -55,8 +81,8 @@ const DEFAULT_BATCH_INSTRUCTIONS = (usedIds: string[]): string =>
|
||||
(usedIds.length
|
||||
? `\n- Identifiants DÉJÀ attribués (réutilise-les pour les mêmes valeurs, n'en duplique AUCUN pour d'autres valeurs) : ${usedIds.join(', ')}.`
|
||||
: '') +
|
||||
'\n- SORTIE STRICTEMENT JSON : {"segments": ["…anonymisé…"], "mapping": {"[PER_1.NOM:M]": "…"}}.' +
|
||||
'\n- "segments" DOIT avoir la même longueur et le même ordre que l’entrée ; "mapping" ne contient que les NOUVELLES entités.';
|
||||
'\n- SORTIE STRICTEMENT JSON : {"segments": ["…anonymisé…"], "mapping": {"[PER_1.NOM:M]": "…"}, "legende": {"PER": "Personne"}}.' +
|
||||
'\n- "segments" DOIT avoir la même longueur et le même ordre que l’entrée ; "mapping" ne contient que les NOUVELLES entités ; "legende" décrit toutes les abréviations utilisées.';
|
||||
|
||||
/**
|
||||
* Build an {@link LlmProvider} backed by any OpenAI-compatible Chat Completions
|
||||
@@ -131,7 +157,11 @@ export function openAICompatibleProvider(opts: OpenAICompatibleOptions): LlmProv
|
||||
|
||||
async anonymize(text: string): Promise<AnonymizationResult> {
|
||||
const content = await chat(systemPrompt, text);
|
||||
const parsed = JSON.parse(content) as { texte_anonymise?: string; mapping?: Record<string, string> };
|
||||
const parsed = JSON.parse(content) as {
|
||||
texte_anonymise?: string;
|
||||
mapping?: Record<string, string>;
|
||||
legende?: Record<string, string>;
|
||||
};
|
||||
if (
|
||||
typeof parsed.texte_anonymise !== 'string' ||
|
||||
typeof parsed.mapping !== 'object' ||
|
||||
@@ -139,20 +169,24 @@ export function openAICompatibleProvider(opts: OpenAICompatibleOptions): LlmProv
|
||||
) {
|
||||
throw new Error('LLM_BAD_SHAPE');
|
||||
}
|
||||
return { anon: parsed.texte_anonymise, mapping: parsed.mapping };
|
||||
return { anon: parsed.texte_anonymise, mapping: parsed.mapping, legend: asLegend(parsed.legende) };
|
||||
},
|
||||
|
||||
async anonymizeBatch(
|
||||
texts: string[],
|
||||
usedIds: string[],
|
||||
): Promise<{ segments: string[]; mapping: Record<string, string> }> {
|
||||
): Promise<{ segments: string[]; mapping: Record<string, string>; legend: Record<string, string> }> {
|
||||
const system = systemPrompt + batchInstructions(usedIds);
|
||||
const content = await chat(system, JSON.stringify({ segments: texts }));
|
||||
const parsed = JSON.parse(content) as { segments?: string[]; mapping?: Record<string, string> };
|
||||
const parsed = JSON.parse(content) as {
|
||||
segments?: string[];
|
||||
mapping?: Record<string, string>;
|
||||
legende?: Record<string, string>;
|
||||
};
|
||||
if (!Array.isArray(parsed.segments) || typeof parsed.mapping !== 'object' || !parsed.mapping) {
|
||||
throw new Error('LLM_BATCH_BAD_SHAPE');
|
||||
}
|
||||
return { segments: parsed.segments, mapping: parsed.mapping };
|
||||
return { segments: parsed.segments, mapping: parsed.mapping, legend: asLegend(parsed.legende) };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
26
src/types.ts
26
src/types.ts
@@ -1,9 +1,17 @@
|
||||
/** Result of an anonymization: text with placeholders + the reverse-mapping table. */
|
||||
/** Result of an anonymization: text with placeholders + the reverse-mapping table + a legend. */
|
||||
export interface AnonymizationResult {
|
||||
/** The text with every detected PII value replaced by a placeholder. */
|
||||
anon: string;
|
||||
/** Placeholder → original value, e.g. `{ "[PER_1.NOM:M]": "Alain JACCARD" }`. */
|
||||
/**
|
||||
* Placeholder → original value, e.g. `{ "[PER_1.NOM:M]": "Alain JACCARD" }`.
|
||||
* SECRET: it re-identifies people. Keep it on your side; never share it downstream.
|
||||
*/
|
||||
mapping: Record<string, string>;
|
||||
/**
|
||||
* Abbreviation → human meaning, e.g. `{ "PER": "Personne", "NOM": "Nom", "M": "Masculin" }`.
|
||||
* NON-secret: safe to send to a downstream LLM so it understands the placeholder tokens.
|
||||
*/
|
||||
legend: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -31,7 +39,7 @@ export interface LlmProvider {
|
||||
anonymizeBatch(
|
||||
texts: string[],
|
||||
usedIds: string[],
|
||||
): Promise<{ segments: string[]; mapping: Record<string, string> }>;
|
||||
): Promise<{ segments: string[]; mapping: Record<string, string>; legend: Record<string, string> }>;
|
||||
}
|
||||
|
||||
/** A structured-PII detector: a tag (e.g. `EMAIL`) and the global regex that finds it. */
|
||||
@@ -45,6 +53,11 @@ export interface PatternDef {
|
||||
* Use it to cut false positives (e.g. a Luhn check on credit-card candidates).
|
||||
*/
|
||||
validate?: (match: string) => boolean;
|
||||
/**
|
||||
* Optional human meaning for this tag, surfaced in the result `legend`
|
||||
* (e.g. `'Adresse e-mail'` for tag `EMAIL`). Falls back to the tag itself.
|
||||
*/
|
||||
meaning?: string;
|
||||
}
|
||||
|
||||
/** Minimal logger sink. Defaults to a no-op; pass your own to capture fallback warnings. */
|
||||
@@ -56,7 +69,12 @@ export interface Logger {
|
||||
export interface AnonymizerConfig {
|
||||
/** LLM backend for free-form PII (names, addresses…). Omit for regex-only mode. */
|
||||
llm?: LlmProvider;
|
||||
/** Structured-PII patterns for the regex fallback. Defaults to {@link presets.swiss}. */
|
||||
/**
|
||||
* Structured-PII patterns for the regex fallback (e.g. {@link presets.swiss}).
|
||||
* Opt-in: omit to disable the fallback — then an LLM failure throws
|
||||
* {@link AnonymizationError} (fail-closed) instead of degrading. At least one of
|
||||
* `llm` or `patterns` must be provided.
|
||||
*/
|
||||
patterns?: PatternDef[];
|
||||
/**
|
||||
* Heuristic that flags likely proper names so the LLM is consulted. Has a
|
||||
|
||||
Reference in New Issue
Block a user