diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dafe4cd2..bbd0f5146 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - In the CSV editor, split a column into several by a delimiter or regular expression, and merge a column with the one next to it using a separator. Both are single undo steps. (#1469) - In the CSV editor, switch the first row between header and data with `Cmd+Shift+H` when auto-detection guesses wrong. Files without a header are no longer saved with a generated header row. (#1469) - In the CSV editor, set the delimiter, quote character, encoding, and line ending by hand from Edit > Set CSV Properties…, then Reload to re-read the file with those settings. (#1469) +- In the CSV editor, choose the escape character (a doubled quote or a backslash) in Set CSV Properties…, so files that escape quotes with a backslash read and save correctly. (#1469) - Add llama.cpp and MLX as local AI providers. Each preset points at the server's default local endpoint and needs no API key, alongside the existing Ollama and custom OpenAI-compatible options. (#1777) ### Fixed diff --git a/Plugins/CSVInspectorPlugin/CSVWriter.swift b/Plugins/CSVInspectorPlugin/CSVWriter.swift index 3b49426df..705575c9b 100644 --- a/Plugins/CSVInspectorPlugin/CSVWriter.swift +++ b/Plugins/CSVInspectorPlugin/CSVWriter.swift @@ -72,9 +72,8 @@ struct CSVWriter { func encodeRow(_ cells: [String]) -> String { let delimiterScalar = UnicodeScalar(dialect.delimiter) let quoteScalar = UnicodeScalar(dialect.quoteChar) + let escapeScalar = UnicodeScalar(dialect.escapeChar) let delimiter = String(delimiterScalar) - let quote = String(quoteScalar) - let doubledQuote = quote + quote var line = "" for (index, field) in cells.enumerated() { @@ -85,8 +84,7 @@ struct CSVWriter { field, delimiterScalar: delimiterScalar, quoteScalar: quoteScalar, - quote: quote, - doubledQuote: doubledQuote + escapeScalar: escapeScalar ) } return line @@ -108,13 +106,19 @@ struct CSVWriter { _ field: String, delimiterScalar: UnicodeScalar, quoteScalar: UnicodeScalar, - quote: String, - doubledQuote: String + escapeScalar: UnicodeScalar ) -> String { let needsQuoting = field.unicodeScalars.contains { scalar in scalar == delimiterScalar || scalar == quoteScalar || scalar == "\n" || scalar == "\r" } guard needsQuoting else { return field } - return quote + field.replacingOccurrences(of: quote, with: doubledQuote) + quote + let quote = String(quoteScalar) + if escapeScalar == quoteScalar { + return quote + field.replacingOccurrences(of: quote, with: quote + quote) + quote + } + let escape = String(escapeScalar) + var body = field.replacingOccurrences(of: escape, with: escape + escape) + body = body.replacingOccurrences(of: quote, with: escape + quote) + return quote + body + quote } } diff --git a/Plugins/TableProPluginKit/CSVDialect.swift b/Plugins/TableProPluginKit/CSVDialect.swift index bc42c6142..ef7eca38a 100644 --- a/Plugins/TableProPluginKit/CSVDialect.swift +++ b/Plugins/TableProPluginKit/CSVDialect.swift @@ -17,6 +17,7 @@ public struct CSVDialect: Equatable, Sendable { public var delimiter: UInt8 public var quoteChar: UInt8 + public var escapeChar: UInt8 public var encoding: String.Encoding public var lineEnding: LineEnding public var hasBom: Bool @@ -30,6 +31,7 @@ public struct CSVDialect: Equatable, Sendable { ) { self.delimiter = delimiter self.quoteChar = quoteChar + self.escapeChar = quoteChar self.encoding = encoding self.lineEnding = lineEnding self.hasBom = hasBom diff --git a/Plugins/TableProPluginKit/CSVStreamingParser.swift b/Plugins/TableProPluginKit/CSVStreamingParser.swift index 0990148f1..705d9d30b 100644 --- a/Plugins/TableProPluginKit/CSVStreamingParser.swift +++ b/Plugins/TableProPluginKit/CSVStreamingParser.swift @@ -10,6 +10,7 @@ public struct CSVStreamingParser: Sendable { public func indexRows(_ bytes: UnsafeBufferPointer) -> [Range] { var ranges: [Range] = [] let quote = dialect.quoteChar + let escape = dialect.escapeChar let delimiter = dialect.delimiter let count = bytes.count var i = bomSkip(in: bytes) @@ -20,8 +21,12 @@ public struct CSVStreamingParser: Sendable { while i < count { let byte = bytes[i] if insideQuotes { + if escape != quote, byte == escape, i + 1 < count { + i += 2 + continue + } if byte == quote { - if i + 1 < count, bytes[i + 1] == quote { + if escape == quote, i + 1 < count, bytes[i + 1] == quote { i += 2 continue } @@ -69,6 +74,7 @@ public struct CSVStreamingParser: Sendable { var fields: [String] = [] var field: [UInt8] = [] let quote = dialect.quoteChar + let escape = dialect.escapeChar let delimiter = dialect.delimiter var insideQuotes = false var i = range.lowerBound @@ -77,8 +83,13 @@ public struct CSVStreamingParser: Sendable { while i < end { let byte = bytes[i] if insideQuotes { + if escape != quote, byte == escape, i + 1 < end { + field.append(bytes[i + 1]) + i += 2 + continue + } if byte == quote { - if i + 1 < end, bytes[i + 1] == quote { + if escape == quote, i + 1 < end, bytes[i + 1] == quote { field.append(quote) i += 2 continue @@ -115,6 +126,7 @@ public struct CSVStreamingParser: Sendable { public func field(_ bytes: UnsafeBufferPointer, range: Range, column: Int) -> String { guard column >= 0 else { return "" } let quote = dialect.quoteChar + let escape = dialect.escapeChar let delimiter = dialect.delimiter var insideQuotes = false var i = range.lowerBound @@ -126,8 +138,13 @@ public struct CSVStreamingParser: Sendable { while i < end { let byte = bytes[i] if insideQuotes { + if escape != quote, byte == escape, i + 1 < end { + if currentColumn == column { field.append(bytes[i + 1]) } + i += 2 + continue + } if byte == quote { - if i + 1 < end, bytes[i + 1] == quote { + if escape == quote, i + 1 < end, bytes[i + 1] == quote { if currentColumn == column { field.append(quote) } i += 2 continue diff --git a/TablePro/Views/Inspector/CSVPropertiesSheet.swift b/TablePro/Views/Inspector/CSVPropertiesSheet.swift index 1173f02a4..aec3bc54e 100644 --- a/TablePro/Views/Inspector/CSVPropertiesSheet.swift +++ b/TablePro/Views/Inspector/CSVPropertiesSheet.swift @@ -13,6 +13,7 @@ struct CSVPropertiesSheet: View { @State private var delimiterIndex: Int @State private var quoteIndex: Int + @State private var escapeIndex: Int @State private var encodingIndex: Int @State private var lineEndingIndex: Int @@ -26,6 +27,7 @@ struct CSVPropertiesSheet: View { self.onCancel = onCancel _delimiterIndex = State(initialValue: CSVPropertyOptions.delimiterIndex(for: dialect.delimiter)) _quoteIndex = State(initialValue: CSVPropertyOptions.quoteIndex(for: dialect.quoteChar)) + _escapeIndex = State(initialValue: CSVPropertyOptions.escapeIndex(for: dialect.escapeChar)) _encodingIndex = State(initialValue: CSVPropertyOptions.encodingIndex(for: dialect.encoding)) _lineEndingIndex = State(initialValue: CSVPropertyOptions.lineEndingIndex(for: dialect.lineEnding)) } @@ -48,6 +50,11 @@ struct CSVPropertiesSheet: View { Text(CSVPropertyOptions.quotes[index].label).tag(index) } } + Picker("Escape character", selection: $escapeIndex) { + ForEach(CSVPropertyOptions.escapes.indices, id: \.self) { index in + Text(CSVPropertyOptions.escapes[index].label).tag(index) + } + } Picker("Encoding", selection: $encodingIndex) { ForEach(CSVPropertyOptions.encodings.indices, id: \.self) { index in Text(CSVPropertyOptions.encodings[index].label).tag(index) @@ -77,6 +84,7 @@ struct CSVPropertiesSheet: View { base: baseDialect, delimiterIndex: delimiterIndex, quoteIndex: quoteIndex, + escapeIndex: escapeIndex, encodingIndex: encodingIndex, lineEndingIndex: lineEndingIndex ) diff --git a/TablePro/Views/Inspector/CSVPropertyOptions.swift b/TablePro/Views/Inspector/CSVPropertyOptions.swift index f0bd28286..e2733a75d 100644 --- a/TablePro/Views/Inspector/CSVPropertyOptions.swift +++ b/TablePro/Views/Inspector/CSVPropertyOptions.swift @@ -21,6 +21,13 @@ enum CSVPropertyOptions { (String(localized: "Single Quote '"), 0x27), ] + static let escapes: [(label: String, byte: UInt8)] = [ + (String(localized: "Doubled Quote"), 0x22), + (String(localized: "Backslash \\"), 0x5C), + ] + + static let backslashEscapeIndex = 1 + static let encodings: [(label: String, encoding: String.Encoding)] = [ ("UTF-8", .utf8), ("UTF-16 LE", .utf16LittleEndian), @@ -43,6 +50,10 @@ enum CSVPropertyOptions { quotes.firstIndex { $0.byte == byte } ?? 0 } + static func escapeIndex(for byte: UInt8) -> Int { + byte == escapes[backslashEscapeIndex].byte ? backslashEscapeIndex : 0 + } + static func encodingIndex(for encoding: String.Encoding) -> Int { encodings.firstIndex { $0.encoding == encoding } ?? 0 } @@ -55,15 +66,20 @@ enum CSVPropertyOptions { base: CSVDialect, delimiterIndex: Int, quoteIndex: Int, + escapeIndex: Int, encodingIndex: Int, lineEndingIndex: Int ) -> CSVDialect { - CSVDialect( + var dialect = CSVDialect( delimiter: delimiters.indices.contains(delimiterIndex) ? delimiters[delimiterIndex].byte : base.delimiter, quoteChar: quotes.indices.contains(quoteIndex) ? quotes[quoteIndex].byte : base.quoteChar, encoding: encodings.indices.contains(encodingIndex) ? encodings[encodingIndex].encoding : base.encoding, lineEnding: lineEndings.indices.contains(lineEndingIndex) ? lineEndings[lineEndingIndex].value : base.lineEnding, hasBom: base.hasBom ) + dialect.escapeChar = escapeIndex == backslashEscapeIndex + ? escapes[backslashEscapeIndex].byte + : dialect.quoteChar + return dialect } } diff --git a/TableProTests/Plugins/CSVInspectorTests.swift b/TableProTests/Plugins/CSVInspectorTests.swift index abef10d9e..d27c578e1 100644 --- a/TableProTests/Plugins/CSVInspectorTests.swift +++ b/TableProTests/Plugins/CSVInspectorTests.swift @@ -77,6 +77,18 @@ struct CSVDialectDetectionTests { let dialect = CSVDialect.detect(from: data) #expect(dialect.encoding == .windowsCP1252) } + + @Test("Default escape character is the quote character") + func defaultEscapeChar() { + #expect(CSVDialect.csv.escapeChar == 0x22) + let detected = CSVDialect.detect(from: "a,b\n1,2\n".data(using: .utf8)!) + #expect(detected.escapeChar == 0x22) + } + + @Test("Escape character follows a non-default quote character") + func escapeFollowsQuote() { + #expect(CSVDialect(delimiter: 0x2C, quoteChar: 0x27).escapeChar == 0x27) + } } @Suite("CSVStreamingParser") @@ -133,6 +145,37 @@ struct CSVStreamingParserTests { #expect(fields == ["a", #"say "hi""#, "c"]) } + @Test("Backslash escape decodes an escaped quote to a single quote") + func backslashEscapesQuote() { + var dialect = CSVDialect.csv + dialect.escapeChar = 0x5C + let (data, ranges, parser) = parse(#""a\"b",c"# + "\n", dialect: dialect) + #expect(ranges.count == 1) + let fields = row(data, parser, ranges[0]) + #expect(fields == [#"a"b"#, "c"]) + } + + @Test("Backslash escape decodes a doubled backslash to a single backslash") + func backslashEscapesBackslash() { + var dialect = CSVDialect.csv + dialect.escapeChar = 0x5C + let (data, ranges, parser) = parse(#""a\\b""# + "\n", dialect: dialect) + let fields = row(data, parser, ranges[0]) + #expect(fields == [#"a\b"#]) + } + + @Test("field(at:column:) honors the backslash escape inside a quoted field") + func fieldBackslashEscape() { + var dialect = CSVDialect.csv + dialect.escapeChar = 0x5C + let (data, ranges, parser) = parse(#""a\"b",c"# + "\n", dialect: dialect) + let first = data.withUnsafeBytes { raw -> String in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return "" } + return parser.field(UnsafeBufferPointer(start: base, count: raw.count), range: ranges[0], column: 0) + } + #expect(first == #"a"b"#) + } + @Test("Empty fields preserved") func emptyFields() { let (data, ranges, parser) = parse(",,,\n") @@ -473,6 +516,20 @@ struct CSVWriterRoundTripTests { #expect(written.prefix(3) == Data([0xEF, 0xBB, 0xBF])) } + @Test("Writer doubles an embedded quote by default") + func writerDefaultDoublesQuote() { + let writer = CSVWriter(dialect: .csv) + #expect(writer.encodeRow([#"a"b"#, "c"]) == #""a""b",c"#) + } + + @Test("Writer escapes an embedded quote with the escape character") + func writerBackslashEscapesQuote() { + var dialect = CSVDialect.csv + dialect.escapeChar = 0x5C + let writer = CSVWriter(dialect: dialect) + #expect(writer.encodeRow([#"a"b"#, "c"]) == #""a\"b",c"#) + } + @Test("A headerless file is written without a synthetic header row") func roundTripHeaderless() throws { let source = "1,2\n3,4\n" diff --git a/TableProTests/Views/CSVPropertyOptionsTests.swift b/TableProTests/Views/CSVPropertyOptionsTests.swift index d3dcfcdd9..495803cbd 100644 --- a/TableProTests/Views/CSVPropertyOptionsTests.swift +++ b/TableProTests/Views/CSVPropertyOptionsTests.swift @@ -22,20 +22,36 @@ struct CSVPropertyOptionsTests { #expect(CSVPropertyOptions.delimiterIndex(for: 0x5E) == 0) } - @Test("Building a dialect from indices sets the four properties and keeps the BOM") + @Test("Building a dialect from indices sets every property and keeps the BOM") func dialectRoundTrip() { let base = CSVDialect(delimiter: 0x2C, hasBom: true) let dialect = CSVPropertyOptions.dialect( base: base, delimiterIndex: CSVPropertyOptions.delimiterIndex(for: 0x09), quoteIndex: CSVPropertyOptions.quoteIndex(for: 0x27), + escapeIndex: CSVPropertyOptions.escapeIndex(for: 0x5C), encodingIndex: CSVPropertyOptions.encodingIndex(for: .utf16LittleEndian), lineEndingIndex: CSVPropertyOptions.lineEndingIndex(for: .cr) ) #expect(dialect.delimiter == 0x09) #expect(dialect.quoteChar == 0x27) + #expect(dialect.escapeChar == 0x5C) #expect(dialect.encoding == .utf16LittleEndian) #expect(dialect.lineEnding == .cr) #expect(dialect.hasBom) } + + @Test("The doubled-quote escape follows the selected quote character") + func doubledQuoteEscapeTracksQuote() { + let dialect = CSVPropertyOptions.dialect( + base: CSVDialect(delimiter: 0x2C), + delimiterIndex: 0, + quoteIndex: CSVPropertyOptions.quoteIndex(for: 0x27), + escapeIndex: 0, + encodingIndex: 0, + lineEndingIndex: 0 + ) + #expect(dialect.quoteChar == 0x27) + #expect(dialect.escapeChar == 0x27) + } } diff --git a/docs/features/csv-inspector.mdx b/docs/features/csv-inspector.mdx index 2a6aa0f10..439e0cda7 100644 --- a/docs/features/csv-inspector.mdx +++ b/docs/features/csv-inspector.mdx @@ -26,7 +26,7 @@ When a file opens, TablePro detects: - **Line ending**: CRLF, LF, or CR, whichever appears first in the first 64 KB. - **Header row**: row one is treated as headers when at least half of its cells are non-empty and not numbers. Otherwise TablePro generates `Column 1`, `Column 2`, ... and treats every row as data. -When a guess is wrong, choose **Edit > Set CSV Properties…** to pick the delimiter, quote character, encoding, and line ending by hand, then **Reload** to re-read the file with those settings. Reload discards unsaved edits and asks first if you have any. +When a guess is wrong, choose **Edit > Set CSV Properties…** to pick the delimiter, quote character, escape character, encoding, and line ending by hand, then **Reload** to re-read the file with those settings. The escape character is either a doubled quote (the RFC 4180 default) or a backslash. Reload discards unsaved edits and asks first if you have any. ## Pagination