feat: initial release of @mobiletic/anonymizer
Framework-agnostic PII anonymization extracted from Mobiletic's chatbot. Pluggable LLM detection + configurable regex fallback, deterministic coreference, and streaming-safe de-anonymization. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Mobiletic
124
test/anonymizer.test.ts
Normal file
124
test/anonymizer.test.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { Anonymizer, presets, type LlmProvider } from '../src/index.js';
|
||||
|
||||
/** An LLM provider that is configured but always fails → forces the regex fallback. */
|
||||
const failingLlm = (overrides: Partial<LlmProvider> = {}): LlmProvider => ({
|
||||
isConfigured: () => true,
|
||||
anonymize: vi.fn().mockRejectedValue(new Error('LLM_NOT_CONFIGURED')),
|
||||
anonymizeBatch: vi.fn().mockRejectedValue(new Error('LLM_NOT_CONFIGURED')),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('Anonymizer (Swiss preset)', () => {
|
||||
const svc = new Anonymizer({ llm: failingLlm(), patterns: presets.swiss });
|
||||
|
||||
it('skips anonymization (no LLM call) when there is no PII', async () => {
|
||||
const llm = failingLlm();
|
||||
const s = new Anonymizer({ llm, patterns: presets.swiss });
|
||||
const r = await s.anonymize('Explique la différence entre INNER JOIN et LEFT JOIN');
|
||||
expect(r.mapping).toEqual({});
|
||||
expect(r.anon).toContain('INNER JOIN');
|
||||
expect(llm.anonymize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls back to regex for structured PII (email/phone/AVS)', async () => {
|
||||
const r = await svc.anonymize('Contact: jean@exemple.ch, +41 79 123 45 67, AVS 756.1234.5678.90');
|
||||
expect(r.anon).not.toContain('jean@exemple.ch');
|
||||
expect(r.anon).not.toContain('756.1234.5678.90');
|
||||
expect(Object.values(r.mapping)).toContain('jean@exemple.ch');
|
||||
expect(svc.deanonymize(r.anon, r.mapping)).toContain('jean@exemple.ch');
|
||||
});
|
||||
|
||||
it('deanonymizes a placeholder split across stream tokens (critical)', () => {
|
||||
const mapping = { '[PER_1.NOM:M]': 'Alain JACCARD' };
|
||||
const d = svc.makeStreamDeanonymizer(mapping);
|
||||
let out = '';
|
||||
out += d.push('Bonjour [PER_');
|
||||
out += d.push('1.NOM');
|
||||
out += d.push(':M], ravi');
|
||||
out += d.flush();
|
||||
expect(out).toBe('Bonjour Alain JACCARD, ravi');
|
||||
expect(out).not.toContain('[PER_');
|
||||
});
|
||||
|
||||
it('streams plain text through unchanged', () => {
|
||||
const d = svc.makeStreamDeanonymizer({});
|
||||
expect(d.push('Un INNER JOIN ') + d.push('retourne...') + d.flush()).toBe('Un INNER JOIN retourne...');
|
||||
});
|
||||
|
||||
describe('anonymizeChunks', () => {
|
||||
it('PII-free chunks → no LLM call, returned as-is', async () => {
|
||||
const llm = failingLlm();
|
||||
const s = new Anonymizer({ llm, patterns: presets.swiss });
|
||||
const r = await s.anonymizeChunks(['Un INNER JOIN combine deux tables.'], {});
|
||||
expect(llm.anonymizeBatch).not.toHaveBeenCalled();
|
||||
expect(r.anon[0]).toContain('INNER JOIN');
|
||||
expect(r.mapping).toEqual({});
|
||||
});
|
||||
|
||||
it('reuses the question placeholder for the same person (deterministic, no LLM)', async () => {
|
||||
const llm = failingLlm();
|
||||
const s = new Anonymizer({ llm, patterns: presets.swiss });
|
||||
const seed = { '[PER_1.NOM:M]': 'Alain JACCARD' };
|
||||
const r = await s.anonymizeChunks(['Le dossier de Alain JACCARD est complet.'], seed);
|
||||
expect(llm.anonymizeBatch).not.toHaveBeenCalled();
|
||||
expect(r.anon[0]).toContain('[PER_1.NOM:M]');
|
||||
expect(r.anon[0]).not.toContain('Alain JACCARD');
|
||||
expect(r.mapping).toEqual(seed);
|
||||
});
|
||||
|
||||
it('renumbers a NEW person that collides with the question placeholder', async () => {
|
||||
const anonymizeBatch = vi.fn().mockResolvedValue({
|
||||
segments: ['[PER_1.NOM:M] a signé.'],
|
||||
mapping: { '[PER_1.NOM:M]': 'Bob Martin' },
|
||||
});
|
||||
const s = new Anonymizer({ llm: failingLlm({ anonymizeBatch }), patterns: presets.swiss });
|
||||
const seed = { '[PER_1.NOM:M]': 'Alain JACCARD' };
|
||||
const r = await s.anonymizeChunks(['Bob Martin a signé.'], seed);
|
||||
expect(anonymizeBatch).toHaveBeenCalledTimes(1);
|
||||
expect(r.anon[0]).toContain('[PER_2.NOM:M]');
|
||||
expect(r.mapping['[PER_1.NOM:M]']).toBe('Alain JACCARD');
|
||||
expect(r.mapping['[PER_2.NOM:M]']).toBe('Bob Martin');
|
||||
});
|
||||
|
||||
it('falls back to regex (structured ids) when the LLM is unavailable', async () => {
|
||||
const s = new Anonymizer({ llm: failingLlm(), patterns: presets.swiss });
|
||||
const r = await s.anonymizeChunks(['Contact : jean@exemple.ch'], {});
|
||||
expect(r.anon[0]).not.toContain('jean@exemple.ch');
|
||||
expect(Object.values(r.mapping)).toContain('jean@exemple.ch');
|
||||
});
|
||||
});
|
||||
|
||||
describe('regex-only mode (no LLM provider)', () => {
|
||||
const s = new Anonymizer({ patterns: presets.swiss });
|
||||
|
||||
it('still anonymizes structured PII without any provider', async () => {
|
||||
const r = await s.anonymize('Écris à jean@exemple.ch');
|
||||
expect(r.anon).not.toContain('jean@exemple.ch');
|
||||
expect(Object.values(r.mapping)).toContain('jean@exemple.ch');
|
||||
});
|
||||
|
||||
it('leaves a bare proper name untouched (no LLM to catch it)', async () => {
|
||||
const r = await s.anonymize('Alain Jaccard a réussi');
|
||||
// The name-hint flags it, but with no LLM the regex fallback finds no structured id.
|
||||
expect(r.anon).toContain('Alain Jaccard');
|
||||
expect(r.mapping).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('configurable presets', () => {
|
||||
it('generic preset anonymizes an IPv4 address', async () => {
|
||||
const s = new Anonymizer({ patterns: presets.generic });
|
||||
const r = await s.anonymize('Serveur 192.168.1.42 indisponible');
|
||||
expect(r.anon).not.toContain('192.168.1.42');
|
||||
expect(Object.values(r.mapping)).toContain('192.168.1.42');
|
||||
});
|
||||
|
||||
it('accepts a fully custom pattern set', async () => {
|
||||
const s = new Anonymizer({ patterns: [{ tag: 'TICKET', re: /\bJIRA-\d+\b/g }] });
|
||||
const r = await s.anonymize('Voir JIRA-123');
|
||||
expect(r.anon).toContain('[TICKET_1]');
|
||||
expect(r.mapping['[TICKET_1]']).toBe('JIRA-123');
|
||||
});
|
||||
});
|
||||
});
|
||||
58
test/openai-compatible.test.ts
Normal file
58
test/openai-compatible.test.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { openAICompatibleProvider } from '../src/index.js';
|
||||
|
||||
const opts = { baseUrl: 'https://api.example.com/v1', apiKey: 'k', model: 'm' };
|
||||
|
||||
function mockFetchJson(content: unknown) {
|
||||
return vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ choices: [{ message: { content: JSON.stringify(content) } }] }),
|
||||
});
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('openAICompatibleProvider', () => {
|
||||
it('isConfigured() reflects whether baseUrl/apiKey/model are present', () => {
|
||||
expect(openAICompatibleProvider(opts).isConfigured()).toBe(true);
|
||||
expect(openAICompatibleProvider({ ...opts, apiKey: '' }).isConfigured()).toBe(false);
|
||||
});
|
||||
|
||||
it('anonymize() parses {texte_anonymise, mapping} and posts temperature 0 + json_object', async () => {
|
||||
const fetchMock = mockFetchJson({ texte_anonymise: '[EMAIL_1]', mapping: { '[EMAIL_1]': 'a@b.ch' } });
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const r = await openAICompatibleProvider(opts).anonymize('a@b.ch');
|
||||
expect(r).toEqual({ anon: '[EMAIL_1]', mapping: { '[EMAIL_1]': 'a@b.ch' } });
|
||||
|
||||
const [url, init] = fetchMock.mock.calls[0];
|
||||
expect(url).toBe('https://api.example.com/v1/chat/completions');
|
||||
const body = JSON.parse(init.body);
|
||||
expect(body.temperature).toBe(0);
|
||||
expect(body.response_format).toEqual({ type: 'json_object' });
|
||||
expect(init.headers.Authorization).toBe('Bearer k');
|
||||
});
|
||||
|
||||
it('anonymizeBatch() returns {segments, mapping} and includes used ids in the prompt', async () => {
|
||||
const fetchMock = mockFetchJson({ segments: ['[PER_2.NOM:M]'], mapping: { '[PER_2.NOM:M]': 'Bob' } });
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const r = await openAICompatibleProvider(opts).anonymizeBatch(['Bob'], ['[PER_1.NOM:M]']);
|
||||
expect(r.segments).toEqual(['[PER_2.NOM:M]']);
|
||||
|
||||
const system = JSON.parse(fetchMock.mock.calls[0][1].body).messages[0].content;
|
||||
expect(system).toContain('[PER_1.NOM:M]');
|
||||
});
|
||||
|
||||
it('throws on a non-OK HTTP status', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, status: 500 }));
|
||||
await expect(openAICompatibleProvider(opts).anonymize('x')).rejects.toThrow('LLM_HTTP_500');
|
||||
});
|
||||
|
||||
it('throws on a malformed response shape', async () => {
|
||||
vi.stubGlobal('fetch', mockFetchJson({ wrong: true }));
|
||||
await expect(openAICompatibleProvider(opts).anonymize('x')).rejects.toThrow('LLM_BAD_SHAPE');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user