From 6392bf4c527fdff3c2e5ceb8abe006cb3fa6203c Mon Sep 17 00:00:00 2001 From: Andrew Skowronski Date: Mon, 20 Jul 2026 16:03:56 -0400 Subject: [PATCH 1/2] [#51] Clear error for duplicate SerializedFile/archive names Analyze only supports a single build at a time. A second SerializedFile or archive with a name already processed used to fail with a raw 'UNIQUE constraint failed' SQLite error (only visible with -v), and two archives sharing a name were silently recorded as duplicate rows. Detect these cases and report a distinctive, self-contained one-line message, skipping the offending file/archive and continuing. Enforce unique archive names at the schema level (bump user_version to 6) and add tests plus docs. --- Analyzer/AnalyzeDuplicateException.cs | 22 +++ Analyzer/AnalyzerTool.cs | 9 ++ Analyzer/Resources/Init.sql | 10 +- .../Commands/SerializedFile/AddArchive.cs | 3 +- .../SQLite/Parsers/SerializedFileParser.cs | 15 +- .../Writers/SerializedFileSQLiteWriter.cs | 25 ++++ Documentation/command-analyze.md | 31 ++-- .../AnalyzeDuplicateNameTests.cs | 136 ++++++++++++++++++ 8 files changed, 232 insertions(+), 19 deletions(-) create mode 100644 Analyzer/AnalyzeDuplicateException.cs create mode 100644 UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs diff --git a/Analyzer/AnalyzeDuplicateException.cs b/Analyzer/AnalyzeDuplicateException.cs new file mode 100644 index 0000000..9faf19a --- /dev/null +++ b/Analyzer/AnalyzeDuplicateException.cs @@ -0,0 +1,22 @@ +using System; + +namespace UnityDataTools.Analyzer; + +// Thrown when analyze encounters a second SerializedFile or archive with a name it has already +// processed. Only a single build can be analyzed at a time: a SerializedFile is referenced by name, +// so two files sharing a name are indistinguishable to Unity's cross-file references. The message +// is self-contained and leads with a distinctive phrase that is documented in command-analyze.md. +public class AnalyzeDuplicateException : Exception +{ + public string DuplicateName { get; } + public bool IsArchive { get; } + + public AnalyzeDuplicateException(string duplicateName, bool isArchive) + : base(isArchive + ? $"Duplicate archive name '{duplicateName}'. Each analyzed archive must have a unique name; only a single build can be analyzed at a time." + : $"Duplicate SerializedFile name '{duplicateName}'. Only a single build can be analyzed at a time; the same SerializedFile name cannot be analyzed twice.") + { + DuplicateName = duplicateName; + IsArchive = isArchive; + } +} diff --git a/Analyzer/AnalyzerTool.cs b/Analyzer/AnalyzerTool.cs index 80ec988..c2204b7 100644 --- a/Analyzer/AnalyzerTool.cs +++ b/Analyzer/AnalyzerTool.cs @@ -121,6 +121,15 @@ public int Analyze(AnalyzeOptions options) Console.Error.WriteLine($"Failed to open: {relativePath}"); countFailures++; } + catch (AnalyzeDuplicateException e) + { + // A file or archive with this name was already analyzed. Only a single build + // can be analyzed at a time; print a clear one-line message (always visible, + // not just with -v) and continue, counting this file as failed. + EraseProgressLine(); + Console.Error.WriteLine($"Skipping {relativePath}: {e.Message}"); + countFailures++; + } catch (Exception e) { // Unexpected failure (SQL error, I/O error, bug, etc.) — print full details. diff --git a/Analyzer/Resources/Init.sql b/Analyzer/Resources/Init.sql index 7cbde59..09f30d2 100644 --- a/Analyzer/Resources/Init.sql +++ b/Analyzer/Resources/Init.sql @@ -8,12 +8,16 @@ CREATE TABLE IF NOT EXISTS types -- Describes a unity archive that contains serialized files and other built content. -- A common use of the unity archive is for AssetBundles but it can also be used for -- Player, Content Archive and ContentDirectory builds. +-- name is UNIQUE: analyze only supports a single build, so two archives with the same name would +-- make queries ambiguous. A duplicate is caught in code and reported (see AnalyzeDuplicateException); +-- the constraint is the durable backstop for that invariant. CREATE TABLE IF NOT EXISTS archives ( id INTEGER, name TEXT, file_size INTEGER, - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE (name) ); -- One row per SerializedFile encountered during analysis. The name is often a technical, @@ -198,8 +202,8 @@ INSERT INTO types (id, name) VALUES (-1, 'Scene'); -- assetbundle_assets/preload_dependencies (issue #82); 3 = renamed asset_bundles table to archives -- and the asset_bundle column/alias to archive (issue #68); 4 = build_report_packed_asset_contents_view -- type column changed from numeric id to type name (issue #55); 5 = added dangling_refs table/view --- (issue #85); databases produced before versioning report 0. -PRAGMA user_version = 5; +-- (issue #85); 6 = archives.name is unique (issue #51); databases produced before versioning report 0. +PRAGMA user_version = 6; PRAGMA synchronous = OFF; PRAGMA journal_mode = MEMORY; diff --git a/Analyzer/SQLite/Commands/SerializedFile/AddArchive.cs b/Analyzer/SQLite/Commands/SerializedFile/AddArchive.cs index 3e0e5cf..d0d9ea6 100644 --- a/Analyzer/SQLite/Commands/SerializedFile/AddArchive.cs +++ b/Analyzer/SQLite/Commands/SerializedFile/AddArchive.cs @@ -9,7 +9,8 @@ create table archives id INTEGER, name TEXT, file_size INTEGER, - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE (name) ); */ internal class AddArchive : AbstractCommand diff --git a/Analyzer/SQLite/Parsers/SerializedFileParser.cs b/Analyzer/SQLite/Parsers/SerializedFileParser.cs index b937d8e..366f347 100644 --- a/Analyzer/SQLite/Parsers/SerializedFileParser.cs +++ b/Analyzer/SQLite/Parsers/SerializedFileParser.cs @@ -122,13 +122,18 @@ void ProcessFile(string file, string rootDirectory) // tracked separately so it isn't lumped with genuine processing errors. archiveHadMissingTypeTrees = true; } + catch (AnalyzeDuplicateException e) + { + // A SerializedFile with this name was already analyzed (e.g. two + // differently-named bundles containing the same CAB). Report the + // self-contained message rather than a raw SQLite constraint error. + Console.Error.WriteLine($"Skipping {node.Path} in archive {archiveName}: {e.Message}"); + archiveHadErrors = true; + } catch (Exception e) { - // the most likely exception here is Microsoft.Data.Sqlite.SqliteException, - // for example 'UNIQUE constraint failed: serialized_files.id'. - // or 'UNIQUE constraint failed: objects.id' which can happen - // if AssetBundles from different builds are being processed by a single call to Analyze - // or if there is a Unity Data Tool bug. + // An unexpected error, for example a Unity Data Tool bug. Duplicate + // names (the common 'UNIQUE constraint failed' case) are handled above. Console.Error.WriteLine($"Error processing {node.Path} in archive {archiveName}"); Console.Error.WriteLine(e.Message); Console.Error.WriteLine(); diff --git a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs index 1eaea3e..b85bd3f 100644 --- a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs +++ b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs @@ -19,6 +19,12 @@ public class SerializedFileSQLiteWriter : IDisposable private int m_CurrentArchiveId = -1; private int m_NextArchiveId = 0; + // Names/ids already written to the archives and serialized_files tables, used to reject a + // second copy of the same content with a clear error instead of a raw UNIQUE constraint + // failure. Only a single build can be analyzed at a time (see AnalyzeDuplicateException). + private HashSet m_WrittenArchiveNames = new(StringComparer.OrdinalIgnoreCase); + private HashSet m_WrittenSerializedFileIds = new(); + private bool m_SkipReferences; private bool m_SkipCrc; @@ -135,7 +141,17 @@ public void BeginArchive(string name, long size) throw new InvalidOperationException("SQLWriter.BeginArchive called twice"); } + // Assign the id before the duplicate check throws, so the caller's EndArchive (called from + // its finally) unwinds cleanly instead of masking the exception. m_CurrentArchiveId = m_NextArchiveId++; + + // Reject a duplicate archive name so no second archives row is created and the caller can + // report it before reading the archive's contents. + if (!m_WrittenArchiveNames.Add(name)) + { + throw new AnalyzeDuplicateException(name, isArchive: true); + } + m_AddArchiveCommand.SetValue("id", m_CurrentArchiveId); m_AddArchiveCommand.SetValue("name", name); m_AddArchiveCommand.SetValue("file_size", size); @@ -170,6 +186,14 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con int serializedFileId = m_SerializedFileIdProvider.GetId(Path.GetFileName(fullPath).ToLowerInvariant()); int sceneId = -1; + // Two SerializedFiles with the same name map to the same id (the provider deduplicates by + // name), so a second one would collide on serialized_files.id. Reject it before opening a + // transaction; the file name is what matters to the user, not the analyzer id. + if (m_WrittenSerializedFileIds.Contains(serializedFileId)) + { + throw new AnalyzeDuplicateException(Path.GetFileName(fullPath), isArchive: false); + } + using var transaction = m_Database.BeginTransaction(); m_CurrentTransaction = transaction; @@ -352,6 +376,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con } transaction.Commit(); + m_WrittenSerializedFileIds.Add(serializedFileId); } catch (Exception) { diff --git a/Documentation/command-analyze.md b/Documentation/command-analyze.md index 664764a..1a8536b 100644 --- a/Documentation/command-analyze.md +++ b/Documentation/command-analyze.md @@ -154,25 +154,36 @@ This error occurs when SerializedFiles are built without TypeTrees. The command UnityDataTool analyze /path/to/bundles --typetree-data /path/to/typetree.bin ``` -### SQL Constraint Errors +### Duplicate SerializedFile name / Duplicate archive name ``` -SQLite Error 19: 'UNIQUE constraint failed: objects.id' +Skipping build2\level0: Duplicate SerializedFile name 'level0'. Only a single build can be analyzed at a time; the same SerializedFile name cannot be analyzed twice. ``` or ``` -SQLite Error 19: 'UNIQUE constraint failed: serialized_files.id'. +Skipping build2\assetbundle: Duplicate archive name 'assetbundle'. Each analyzed archive must have a unique name; only a single build can be analyzed at a time. ``` -These errors occur when the same serialized file name appears in multiple sources: +**analyze only supports a single build at a time.** Unity resolves references between SerializedFiles +by file name, so two files that share a name are indistinguishable to those references — there is no +way to tell which copy a reference points at. For that reason each SerializedFile name (and each +archive name) may appear only once in a database. -| Cause | Solution | -|-------|----------| -| Multiple builds in same directory | Analyze each build separately | -| Scenes with same filename (different paths) | Rename scenes to be unique | -| AssetBundle variants | Analyze variants separately | +When analyze encounters a second file or archive with a name it has already processed, it prints one +of the messages above, **skips that file or archive** (counting it as a failed file), and continues +with the rest of the input. The already-analyzed copy is kept; the duplicate's content is ignored. -See [Comparing Builds](../../Documentation/comparing-builds.md) for strategies to compare different versions of builds. +This is expected when the input contains more than one build, and in these common cases: + +| Cause | What to do | +|-------|------------| +| Multiple builds passed together (or nested in one directory) | Analyze each build into its own database | +| AssetBundle variants (same content, different variant) | Analyze each variant separately | +| Hashed AssetBundle file names across two builds | The file names differ but the inner SerializedFile (`CAB-`) is shared — analyze each build separately | +| Player scenes with the same file name (`level0`, …) from different builds | Analyze each build separately | + +To compare two builds, analyze each into a separate database and query across them — see +[Comparing Builds](../../Documentation/comparing-builds.md). ### Slow Analyze times, large output database diff --git a/UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs b/UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs new file mode 100644 index 0000000..c9d2001 --- /dev/null +++ b/UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs @@ -0,0 +1,136 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using NUnit.Framework; + +namespace UnityDataTools.UnityDataTool.Tests; + +#pragma warning disable NUnit2005, NUnit2006 + +// Tests the duplicate-name handling (issue #51): analyze supports only a single build, so a second +// SerializedFile or archive with a name that was already processed is skipped with a clear +// single-line message instead of a raw "UNIQUE constraint failed" SQLite error. Covers the three +// scenarios from the issue: loose files, archives with the same name, and differently-named +// archives (hashed bundle names) that share the same inner SerializedFile. +public class AnalyzeDuplicateNameTests +{ + private string m_TestOutputFolder; + private string m_AssetBundlesFolder; + + [OneTimeSetUp] + public void OneTimeSetup() + { + m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "duplicate_name_test_folder"); + m_AssetBundlesFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", "AssetBundles"); + Directory.CreateDirectory(m_TestOutputFolder); + Directory.SetCurrentDirectory(m_TestOutputFolder); + } + + [TearDown] + public void Teardown() + { + SqliteConnection.ClearAllPools(); + var testDir = new DirectoryInfo(m_TestOutputFolder); + testDir.EnumerateFiles().ToList().ForEach(f => f.Delete()); + testDir.EnumerateDirectories().ToList().ForEach(d => d.Delete(true)); + } + + // Runs analyze and returns its exit code plus whatever it wrote to stderr (where the + // duplicate messages are printed). + private static async Task<(int exitCode, string stderr)> RunAnalyze(params string[] args) + { + var originalError = System.Console.Error; + using var sw = new StringWriter(); + try + { + System.Console.SetError(sw); + var exitCode = await Program.Main(new[] { "analyze" }.Concat(args).ToArray()); + return (exitCode, sw.ToString()); + } + finally + { + System.Console.SetError(originalError); + } + } + + // Case 2 / Case 3: two build folders each contain an archive named "assetbundle" (and "scenes"). + // The second archive of each name is rejected before it is opened, so exactly one row per name + // survives and no UNIQUE constraint error is shown. + [Test] + public async Task Analyze_ArchivesWithSameName_SkippedWithClearMessage() + { + var build1 = Path.Combine(m_AssetBundlesFolder, "2019.4.0f1"); + var build2 = Path.Combine(m_AssetBundlesFolder, "2020.3.0f1"); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stderr) = await RunAnalyze(build1, build2, "-o", databasePath); + + Assert.AreEqual(0, exitCode, "analyze should continue and exit 0 after skipping duplicates"); + StringAssert.Contains("Duplicate archive name 'assetbundle'", stderr); + StringAssert.DoesNotContain("UNIQUE constraint", stderr); + + using var db = SQLTestHelper.OpenDatabase(databasePath); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM archives WHERE name = 'assetbundle'", + 1, "only one archive named 'assetbundle' should be recorded"); + } + + // Case 1: two loose SerializedFiles with the same name in different folders. The duplicate is + // rejected before its transaction is opened, so only the first copy is recorded. + [Test] + public async Task Analyze_LooseFilesWithSameName_SkippedWithClearMessage() + { + var source = Path.Combine(TestContext.CurrentContext.TestDirectory, + "Data", "PlayerWithTypeTrees", "level0"); + var build1 = Path.Combine(m_TestOutputFolder, "build1"); + var build2 = Path.Combine(m_TestOutputFolder, "build2"); + Directory.CreateDirectory(build1); + Directory.CreateDirectory(build2); + File.Copy(source, Path.Combine(build1, "level0")); + File.Copy(source, Path.Combine(build2, "level0")); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stderr) = await RunAnalyze(m_TestOutputFolder, "-o", databasePath); + + Assert.AreEqual(0, exitCode, "analyze should continue and exit 0 after skipping duplicates"); + StringAssert.Contains("Duplicate SerializedFile name 'level0'", stderr); + StringAssert.DoesNotContain("UNIQUE constraint", stderr); + + using var db = SQLTestHelper.OpenDatabase(databasePath); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM serialized_files WHERE name = 'level0'", + 1, "only one SerializedFile named 'level0' should be recorded"); + } + + // Hashed-name shape: the same archive under two different file names (as with hashed bundle + // names). The archive names differ, so both are recorded, but they share the same inner + // SerializedFile ("CAB-"), which is rejected the second time. + [Test] + public async Task Analyze_DifferentArchiveNamesSharingSerializedFile_SkippedWithClearMessage() + { + var source = Path.Combine(m_AssetBundlesFolder, "2019.4.0f1", "assetbundle"); + var bundleA = Path.Combine(m_TestOutputFolder, "bundleA"); + var bundleB = Path.Combine(m_TestOutputFolder, "bundleB"); + File.Copy(source, bundleA); + File.Copy(source, bundleB); + var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder); + + var (exitCode, stderr) = await RunAnalyze(bundleA, bundleB, "-o", databasePath); + + Assert.AreEqual(0, exitCode, "analyze should continue and exit 0 after skipping duplicates"); + StringAssert.Contains("Duplicate SerializedFile name", stderr); + StringAssert.DoesNotContain("UNIQUE constraint", stderr); + + using var db = SQLTestHelper.OpenDatabase(databasePath); + SQLTestHelper.AssertQueryInt(db, + "SELECT COUNT(*) FROM archives WHERE name IN ('bundleA', 'bundleB')", + 2, "both differently-named archives should be recorded"); + // The shared inner SerializedFile is analyzed once; count only files that actually have + // objects, since references also create name-only stub rows for un-analyzed files. + SQLTestHelper.AssertQueryInt(db, + @"SELECT COUNT(*) FROM serialized_files + WHERE name LIKE 'CAB-%' AND id IN (SELECT serialized_file FROM objects)", + 1, "the shared inner SerializedFile should be analyzed only once"); + } +} From 2e74649ddf26456c2a1886c239e359f3ed15488c Mon Sep 17 00:00:00 2001 From: Andrew Skowronski Date: Tue, 21 Jul 2026 09:02:17 -0400 Subject: [PATCH 2/2] [#51] Address review: case-sensitive archive names, fix doc link and comments - Compare archive names case-sensitively so the code matches the case-sensitive archives.name UNIQUE constraint and the name as it exists on the file system (drops the OrdinalIgnoreCase set). - Fix the comparing-builds.md link to be same-folder relative. - Clarify the schema comment and a test comment. --- Analyzer/Resources/Init.sql | 7 ++++--- Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs | 4 +++- Documentation/command-analyze.md | 2 +- UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Analyzer/Resources/Init.sql b/Analyzer/Resources/Init.sql index 09f30d2..43f6652 100644 --- a/Analyzer/Resources/Init.sql +++ b/Analyzer/Resources/Init.sql @@ -8,9 +8,10 @@ CREATE TABLE IF NOT EXISTS types -- Describes a unity archive that contains serialized files and other built content. -- A common use of the unity archive is for AssetBundles but it can also be used for -- Player, Content Archive and ContentDirectory builds. --- name is UNIQUE: analyze only supports a single build, so two archives with the same name would --- make queries ambiguous. A duplicate is caught in code and reported (see AnalyzeDuplicateException); --- the constraint is the durable backstop for that invariant. +-- name is UNIQUE (case-sensitive, matching the name on the file system): analyze only supports a +-- single build, so two archives with the same name would make queries ambiguous. A duplicate is +-- caught in code and reported (see AnalyzeDuplicateException); the constraint is the durable +-- backstop for that invariant. CREATE TABLE IF NOT EXISTS archives ( id INTEGER, diff --git a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs index b85bd3f..bed4d76 100644 --- a/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs +++ b/Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs @@ -22,7 +22,9 @@ public class SerializedFileSQLiteWriter : IDisposable // Names/ids already written to the archives and serialized_files tables, used to reject a // second copy of the same content with a clear error instead of a raw UNIQUE constraint // failure. Only a single build can be analyzed at a time (see AnalyzeDuplicateException). - private HashSet m_WrittenArchiveNames = new(StringComparer.OrdinalIgnoreCase); + // Archive names are compared case-sensitively, matching the archives.name schema constraint + // and the name as it exists on the file system. + private HashSet m_WrittenArchiveNames = new(); private HashSet m_WrittenSerializedFileIds = new(); private bool m_SkipReferences; diff --git a/Documentation/command-analyze.md b/Documentation/command-analyze.md index 1a8536b..c15a8ee 100644 --- a/Documentation/command-analyze.md +++ b/Documentation/command-analyze.md @@ -183,7 +183,7 @@ This is expected when the input contains more than one build, and in these commo | Player scenes with the same file name (`level0`, …) from different builds | Analyze each build separately | To compare two builds, analyze each into a separate database and query across them — see -[Comparing Builds](../../Documentation/comparing-builds.md). +[Comparing Builds](comparing-builds.md). ### Slow Analyze times, large output database diff --git a/UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs b/UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs index c9d2001..665bfdb 100644 --- a/UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs +++ b/UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs @@ -55,8 +55,8 @@ public void Teardown() } // Case 2 / Case 3: two build folders each contain an archive named "assetbundle" (and "scenes"). - // The second archive of each name is rejected before it is opened, so exactly one row per name - // survives and no UNIQUE constraint error is shown. + // The second archive of each name is rejected before it is recorded (its contents are never + // processed), so exactly one row per name survives and no UNIQUE constraint error is shown. [Test] public async Task Analyze_ArchivesWithSameName_SkippedWithClearMessage() {