Fix header metadata: version, package name, checksum, and redundant upstream headers

src/compiler/FilterCompiler.test.ts

@@ -219,3 +219,23 @@ Deno.test('FilterCompiler - should add source header for each source', async ()
     assertEquals(result.some((line) => line.includes('Source name: Test Source')), true);
     assertEquals(result.some((line) => line.includes('Source:')), true);
 });
+
+// Test checksum generation
+Deno.test('FilterCompiler - should add checksum to compiled output', async () => {
+    const compiler = new FilterCompiler(silentLogger);
+    const config = createTestConfig();
+
+    const result = await compiler.compile(config);
+
+    // Should have a checksum line
+    assertEquals(result.some((line) => line.startsWith('! Checksum:')), true);
+
+    // Checksum should be in the header section (before source headers)
+    const checksumIndex = result.findIndex((line) => line.startsWith('! Checksum:'));
+    const firstSourceIndex = result.findIndex((line) => line.includes('Source name:') || line.includes('! Source:'));
+    assertEquals(checksumIndex > 0, true);
+    // If there's a source header, checksum should be before it
+    if (firstSourceIndex > 0) {
+        assertEquals(checksumIndex < firstSourceIndex, true);
+    }
+});

src/compiler/FilterCompiler.ts

@@ -2,7 +2,7 @@ import { IConfiguration, ILogger, ISource, TransformationType, ICompilerEvents }
 import { ConfigurationValidator } from '../configuration/index.ts';
 import { TransformationPipeline } from '../transformations/index.ts';
 import { SourceCompiler } from './SourceCompiler.ts';
-import { logger as defaultLogger, BenchmarkCollector, CompilationMetrics, createEventEmitter, CompilerEventEmitter } from '../utils/index.ts';
+import { logger as defaultLogger, BenchmarkCollector, CompilationMetrics, createEventEmitter, CompilerEventEmitter, addChecksumToHeader } from '../utils/index.ts';
 
 /**
  * Result of compilation with optional metrics.
@@ -18,7 +18,7 @@ export interface CompilationResult {
  */
 const PACKAGE_INFO = {
     name: '@jk-com/adblock-compiler',
-    version: '2.0.0',
+    version: '0.6.88',
 } as const;
 
 /**
@@ -186,7 +186,11 @@ export class FilterCompiler {
         const header = this.prepareHeader(configuration);
         this.logger.info(`Final length of the list is ${header.length + finalList.length}`);
 
-        const rules = [...header, ...finalList];
+        let rules = [...header, ...finalList];
+
+        // Add checksum to the header
+        rules = await addChecksumToHeader(rules);
+
         collector?.setOutputRuleCount(rules.length);
 
         const metrics = collector?.finish();

src/compiler/HeaderGenerator.ts

@@ -5,8 +5,8 @@ import type { IConfiguration, ISource } from '../types/index.ts';
  * Version matches deno.json for JSR publishing.
  */
 const PACKAGE_INFO = {
-    name: '@jk-com/hostlistcompiler',
-    version: '2.0.0',
+    name: '@jk-com/adblock-compiler',
+    version: '0.6.88',
 } as const;
 
 /**

src/compiler/SourceCompiler.ts

@@ -1,7 +1,7 @@
 import { FilterDownloader } from '../downloader/index.ts';
 import { ILogger, ISource, TransformationType } from '../types/index.ts';
 import { TransformationPipeline } from '../transformations/index.ts';
-import { logger as defaultLogger, CompilerEventEmitter, createEventEmitter } from '../utils/index.ts';
+import { logger as defaultLogger, CompilerEventEmitter, createEventEmitter, stripUpstreamHeaders } from '../utils/index.ts';
 
 /**
  * Compiles an individual source according to its configuration.
@@ -54,6 +54,10 @@ export class SourceCompiler {
 
             this.logger.info(`Original length is ${rules.length}`);
 
+            // Strip upstream metadata headers to avoid redundancy
+            rules = stripUpstreamHeaders(rules);
+            this.logger.info(`Length after stripping upstream headers is ${rules.length}`);
+
             // Apply transformations
             const transformations = source.transformations || [];
             rules = await this.pipeline.transform(

src/platform/WorkerCompiler.ts

@@ -8,7 +8,7 @@ import type { IConfiguration, ILogger, ISource, TransformationType, ICompilerEve
 import type { IContentFetcher, IPlatformCompilerOptions } from './types.ts';
 import { ConfigurationValidator } from '../configuration/index.ts';
 import { TransformationPipeline } from '../transformations/index.ts';
-import { silentLogger, createEventEmitter, CompilerEventEmitter, BenchmarkCollector, CompilationMetrics } from '../utils/index.ts';
+import { silentLogger, createEventEmitter, CompilerEventEmitter, BenchmarkCollector, CompilationMetrics, addChecksumToHeader, stripUpstreamHeaders } from '../utils/index.ts';
 import { HttpFetcher } from './HttpFetcher.ts';
 import { PreFetchedContentFetcher } from './PreFetchedContentFetcher.ts';
 import { CompositeFetcher } from './CompositeFetcher.ts';
@@ -27,7 +27,7 @@ export interface WorkerCompilationResult {
  */
 const PACKAGE_INFO = {
     name: '@jk-com/adblock-compiler',
-    version: '2.0.0',
+    version: '0.6.88',
 } as const;
 
 /**
@@ -144,7 +144,10 @@ export class WorkerCompiler {
 
                         const startTime = performance.now();
                         try {
-                            const rules = await downloader.download(source.source);
+                            let rules = await downloader.download(source.source);
+
+                            // Strip upstream metadata headers to avoid redundancy
+                            rules = stripUpstreamHeaders(rules);
 
                             // Apply source-level transformations
                             const transformedRules = await this.applySourceTransformations(
@@ -186,7 +189,11 @@ export class WorkerCompiler {
 
                     const startTime = performance.now();
                     try {
-                        const rules = await downloader.download(source.source);
+                        let rules = await downloader.download(source.source);
+
+                        // Strip upstream metadata headers to avoid redundancy
+                        rules = stripUpstreamHeaders(rules);
+
                         const transformedRules = await this.applySourceTransformations(
                             rules,
                             source,
@@ -261,7 +268,11 @@ export class WorkerCompiler {
 
         // Prepend header
         const header = this.prepareHeader(configuration);
-        const rules = [...header, ...finalList];
+        let rules = [...header, ...finalList];
+
+        // Add checksum to the header
+        rules = await addChecksumToHeader(rules);
+
         collector?.setOutputRuleCount(rules.length);
 
         const metrics = collector?.finish();

src/utils/checksum.test.ts

@@ -0,0 +1,137 @@
+/**
+ * Tests for checksum utilities
+ */
+
+import { assertEquals, assertExists } from '@std/assert';
+import { calculateChecksum, addChecksumToHeader } from './checksum.ts';
+
+Deno.test('calculateChecksum - should generate a checksum for simple content', async () => {
+    const lines = [
+        '||example.com^',
+        '||test.com^',
+    ];
+
+    const checksum = await calculateChecksum(lines);
+
+    assertExists(checksum);
+    assertEquals(checksum.length, 27);
+    // Checksum should be Base64 encoded
+    assertEquals(/^[A-Za-z0-9+/=]+$/.test(checksum), true);
+});
+
+Deno.test('calculateChecksum - should be deterministic', async () => {
+    const lines = [
+        '! Title: Test',
+        '||example.com^',
+        '||test.com^',
+    ];
+
+    const checksum1 = await calculateChecksum(lines);
+    const checksum2 = await calculateChecksum(lines);
+
+    assertEquals(checksum1, checksum2);
+});
+
+Deno.test('calculateChecksum - should ignore existing checksum lines', async () => {
+    const linesWithChecksum = [
+        '! Title: Test',
+        '! Checksum: oldchecksum123',
+        '||example.com^',
+    ];
+
+    const linesWithoutChecksum = [
+        '! Title: Test',
+        '||example.com^',
+    ];
+
+    const checksum1 = await calculateChecksum(linesWithChecksum);
+    const checksum2 = await calculateChecksum(linesWithoutChecksum);
+
+    assertEquals(checksum1, checksum2);
+});
+
+Deno.test('calculateChecksum - should produce different checksums for different content', async () => {
+    const lines1 = ['||example.com^'];
+    const lines2 = ['||test.com^'];
+
+    const checksum1 = await calculateChecksum(lines1);
+    const checksum2 = await calculateChecksum(lines2);
+
+    assertEquals(checksum1 !== checksum2, true);
+});
+
+Deno.test('addChecksumToHeader - should add checksum before Compiled by line', async () => {
+    const lines = [
+        '!',
+        '! Title: Test Filter',
+        '! Last modified: 2026-01-04T00:00:00.000Z',
+        '!',
+        '! Compiled by @jk-com/adblock-compiler v0.6.88',
+        '!',
+        '||example.com^',
+    ];
+
+    const result = await addChecksumToHeader(lines);
+
+    // Should have one more line than original
+    assertEquals(result.length, lines.length + 1);
+
+    // Find the checksum line
+    const checksumLine = result.find(line => line.startsWith('! Checksum:'));
+    assertExists(checksumLine);
+
+    // Checksum should be before "Compiled by"
+    const checksumIndex = result.indexOf(checksumLine!);
+    const compiledByIndex = result.findIndex(line => line.includes('Compiled by'));
+    assertEquals(checksumIndex < compiledByIndex, true);
+});
+
+Deno.test('addChecksumToHeader - should add checksum with correct format', async () => {
+    const lines = [
+        '! Title: Test',
+        '||example.com^',
+    ];
+
+    const result = await addChecksumToHeader(lines);
+
+    const checksumLine = result.find(line => line.startsWith('! Checksum:'));
+    assertExists(checksumLine);
+
+    // Should match format: "! Checksum: <base64>"
+    assertEquals(/^! Checksum: [A-Za-z0-9+/=]{27}$/.test(checksumLine!), true);
+});
+
+Deno.test('addChecksumToHeader - should handle lists without Compiled by line', async () => {
+    const lines = [
+        '! Title: Test',
+        '||example.com^',
+    ];
+
+    const result = await addChecksumToHeader(lines);
+
+    // Should have checksum added
+    const checksumLine = result.find(line => line.startsWith('! Checksum:'));
+    assertExists(checksumLine);
+});
+
+Deno.test('addChecksumToHeader - calculated checksum should validate the content', async () => {
+    const lines = [
+        '! Title: Test',
+        '! Last modified: 2026-01-04T00:00:00.000Z',
+        '!',
+        '! Compiled by @jk-com/adblock-compiler v0.6.88',
+        '!',
+        '||example.com^',
+    ];
+
+    const withChecksum = await addChecksumToHeader(lines);
+
+    // Extract the checksum
+    const checksumLine = withChecksum.find(line => line.startsWith('! Checksum:'));
+    const extractedChecksum = checksumLine!.replace('! Checksum: ', '');
+
+    // Calculate checksum of the result (which should ignore the checksum line)
+    const recalculated = await calculateChecksum(withChecksum);
+
+    assertEquals(extractedChecksum, recalculated);
+});

src/utils/checksum.ts

@@ -0,0 +1,87 @@
+/**
+ * Checksum calculation utilities for filter lists.
+ * Implements a secure checksum format using SHA-256 (instead of legacy MD5).
+ */
+
+/**
+ * Calculates a checksum for a filter list.
+ * The checksum is a Base64-encoded SHA-256 hash of the filter content.
+ * 
+ * Implementation follows standard checksum practices:
+ * 1. Join all lines (except the checksum line itself) with \n
+ * 2. Calculate SHA-256 hash of the content
+ * 3. Encode the hash as Base64
+ * 4. Truncate to 27 characters for consistency with traditional MD5 checksums
+ * 
+ * Note: We use SHA-256 instead of MD5 for better security, truncated to match
+ * the traditional checksum length (MD5 base64 is exactly 24 chars, we use 27).
+ * 
+ * @param lines - Array of filter list lines
+ * @returns Base64-encoded SHA-256 checksum (truncated to 27 chars)
+ */
+export async function calculateChecksum(lines: string[]): Promise<string> {
+    // Filter out any existing checksum lines and join with newlines
+    const content = lines
+        .filter(line => !line.startsWith('! Checksum:'))
+        .join('\n');
+
+    // Convert string to Uint8Array
+    const encoder = new TextEncoder();
+    const data = encoder.encode(content);
+
+    // Calculate SHA-256 hash using Web Crypto API
+    const hashBuffer = await crypto.subtle.digest('SHA-256', data);
+
+    // Convert hash to Base64
+    const hashArray = Array.from(new Uint8Array(hashBuffer));
+    const base64 = btoa(String.fromCharCode(...hashArray));
+
+    // Truncate to 27 characters for consistency
+    return base64.substring(0, 27);
+}
+
+/**
+ * Adds a checksum line to the filter list header.
+ * The checksum line should be inserted after the other metadata
+ * and before the "Compiled by" line.
+ * 
+ * @param lines - Array of filter list lines
+ * @returns Array with checksum line added
+ */
+export async function addChecksumToHeader(lines: string[]): Promise<string[]> {
+    // Calculate checksum of all lines
+    const checksum = await calculateChecksum(lines);
+
+    // Find where to insert the checksum line
+    // It should go after metadata (Title, Description, etc.) but before "Compiled by"
+    const compiledByIndex = lines.findIndex(line => line.includes('! Compiled by'));
+
+    if (compiledByIndex === -1) {
+        // If no "Compiled by" line, add at the end of header section
+        // Look for the first non-comment line or end of file
+        const firstRuleIndex = lines.findIndex(line => 
+            line.trim() !== '' && 
+            !line.startsWith('!')
+        );
+
+        const insertIndex = firstRuleIndex === -1 ? lines.length : firstRuleIndex;
+        return [
+            ...lines.slice(0, insertIndex),
+            `! Checksum: ${checksum}`,
+            ...lines.slice(insertIndex),
+        ];
+    }
+
+    // Insert checksum before "Compiled by" line
+    // Find the blank line before "Compiled by" or the "Compiled by" line itself
+    let insertIndex = compiledByIndex;
+    if (compiledByIndex > 0 && lines[compiledByIndex - 1] === '!') {
+        insertIndex = compiledByIndex - 1;
+    }
+
+    return [
+        ...lines.slice(0, insertIndex),
+        `! Checksum: ${checksum}`,
+        ...lines.slice(insertIndex),
+    ];
+}