diff --git a/README.md b/README.md index ebb0d8f..a699aef 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ docker run --rm -v $(pwd):/data dbpedia/databus-python-client download $DOWNLOAD - `--databus-key` - If the databus is protected and needs API key authentication, you can provide the API key with `--databus-key YOUR_API_KEY`. - `--compression` - - Enables on-the-fly compression format conversion during download. Supported formats: `bz2`, `gz`, `xz`. The source compression is auto-detected from the file extension. Example: `--compression gz` converts all downloaded compressed files to gzip format. + - Enables on-the-fly compression format conversion during download. Supported formats: `bz2`, `gz`, `xz`, `none`. The source compression is auto-detected from the file extension. Use `none` to decompress files without recompressing. Example: `--compression gz` converts all downloaded compressed files to gzip format. - `--format` - Enables on-the-fly RDF and tabular format conversion during download (Layer 2 and Layer 3). Supported formats: `ntriples` (`nt`), `turtle` (`ttl`), `rdf-xml` (`rdf`, `xml`), `nquads` (`nq`), `trig`, `trix`, `json-ld` (`jsonld`), `csv`, `tsv`. Short aliases shown in brackets. Only the converted output file is kept — the original is deleted after successful conversion. Within the same equivalence class (e.g. turtle to ntriples) conversion is lossless. Across classes (e.g. RDF to CSV) some flags below may be required. - `--graph-name` @@ -284,6 +284,9 @@ databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased # Download a collection and unify all files to bz2 format databusclient download https://databus.dbpedia.org/dbpedia/collections/dbpedia-snapshot-2022-12 --compression bz2 + +# Decompress files without recompressing +databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals/2022.12.01/mappingbased-literals_lang=az.ttl.bz2 --compression none ``` **Download with Format Conversion**: download files and convert RDF or tabular format on-the-fly. Only the converted output file is kept. diff --git a/databusclient/api/download.py b/databusclient/api/download.py index fcb5781..caac28f 100644 --- a/databusclient/api/download.py +++ b/databusclient/api/download.py @@ -64,12 +64,15 @@ def _should_convert_compression( """Determine if a file should have its compression format converted or compressed. Source compression is detected automatically from the file extension. + If compression='none', compressed files are decompressed and saved without + any compression. If the file is already uncompressed and compression='none', + nothing is done. If the file is uncompressed and a target compression is specified, it will be compressed to the target format (source_format returned as None). Args: filename: Name of the file. - compression: Target compression format ('bz2', 'gz', 'xz') or None. + compression: Target compression format ('bz2', 'gz', 'xz', 'none') or None. Returns: Tuple of (should_convert: bool, source_format: Optional[str]). @@ -80,7 +83,14 @@ def _should_convert_compression( source_format = _detect_compression_format(filename) - # If file is not compressed, don't convert + # 'none' means decompress — only meaningful if file is compressed + if compression.lower() == "none": + if source_format is None: + # Already uncompressed, nothing to do + return False, None + return True, source_format + + # If file is not compressed, compress it to the target format if source_format is None: return True, None @@ -99,12 +109,21 @@ def _get_converted_filename( Args: filename: Original filename. source_format: Source compression format ('bz2', 'gz', 'xz'). - target_format: Target compression format ('bz2', 'gz', 'xz'). + target_format: Target compression format ('bz2', 'gz', 'xz') or 'none' + to decompress without recompressing. Returns: - New filename with updated extension. + New filename with updated extension. If target_format is 'none', + the compression extension is stripped and nothing is added. """ source_ext = COMPRESSION_EXTENSIONS[source_format] + + # 'none' means decompress — strip compression extension, add nothing + if target_format.lower() == "none": + if filename.lower().endswith(source_ext): + return filename[: -len(source_ext)] + return filename + target_ext = COMPRESSION_EXTENSIONS[target_format] # Handle case-insensitive extension matching @@ -116,36 +135,57 @@ def _get_converted_filename( def _convert_compression_format( source_file: str, target_file: str, source_format: str, target_format: str ) -> None: - """Convert a compressed file from one format to another. + """Convert or decompress a compressed file. + + Handles two cases: + - target_format is 'none': decompress source_file to target_file without recompressing. + - target_format is a compression format: decompress then recompress to target format. Args: source_file: Path to source compressed file. - target_file: Path to target compressed file. + target_file: Path to target file. source_format: Source compression format ('bz2', 'gz', 'xz'). - target_format: Target compression format ('bz2', 'gz', 'xz'). + target_format: Target compression format ('bz2', 'gz', 'xz') or 'none' to decompress only. Raises: - ValueError: If source_format or target_format is not supported. - RuntimeError: If compression conversion fails. + ValueError: If source_format is not supported. + RuntimeError: If the operation fails. """ - # Validate compression formats if source_format not in COMPRESSION_MODULES: raise ValueError( - f"Unsupported source compression format: {source_format}. Supported formats: {list(COMPRESSION_MODULES.keys())}" + f"Unsupported source compression format: {source_format}. " + f"Supported formats: {list(COMPRESSION_MODULES.keys())}" ) + + source_module = COMPRESSION_MODULES[source_format] + + # Decompression-only path: target_format == 'none' + if target_format.lower() == "none": + print(f"Decompressing {os.path.basename(source_file)} -> {os.path.basename(target_file)}") + try: + with source_module.open(source_file, "rb") as sf: + with open(target_file, "wb") as tf: + shutil.copyfileobj(sf, tf) + os.remove(source_file) + print(f"Decompression complete: {os.path.basename(target_file)}") + except Exception as e: + if os.path.exists(target_file): + os.remove(target_file) + raise RuntimeError(f"Decompression failed: {e}") + return + if target_format not in COMPRESSION_MODULES: raise ValueError( - f"Unsupported target compression format: {target_format}. Supported formats: {list(COMPRESSION_MODULES.keys())}" + f"Unsupported target compression format: {target_format}. " + f"Supported formats: {list(COMPRESSION_MODULES.keys())}" ) - source_module = COMPRESSION_MODULES[source_format] target_module = COMPRESSION_MODULES[target_format] print( f"Converting {source_format} → {target_format}: {os.path.basename(source_file)}" ) - # Decompress and recompress with progress indication chunk_size = 8192 try: @@ -561,6 +601,11 @@ def _download_file( shutil.copyfileobj(sf, tf) os.remove(filename) print(f"Compression complete: {os.path.basename(target_filepath)}") + elif compression.lower() == "none": + # Decompress — strip compression extension, save plain file. + target_filename = _get_converted_filename(file, source_fmt, "none") + target_filepath = os.path.join(localDir, target_filename) + _convert_compression_format(filename, target_filepath, source_fmt, "none") else: target_filename = _get_converted_filename(file, source_fmt, compression) target_filepath = os.path.join(localDir, target_filename) @@ -707,10 +752,11 @@ def _download_file( # 4. Source was NOT compressed, no --compression given -> no compression if source_compression is not None: if should_convert_compression and compression: - final_compression = compression + # 'none' means no recompression after format conversion + final_compression = None if compression.lower() == "none" else compression else: final_compression = source_compression - elif compression: + elif compression and compression.lower() != "none": # Source was uncompressed but user explicitly requested --compression final_compression = compression else: diff --git a/databusclient/cli.py b/databusclient/cli.py index f64c128..14c03a1 100644 --- a/databusclient/cli.py +++ b/databusclient/cli.py @@ -251,10 +251,10 @@ def _write_manifest(): @click.option( "--compression", "compression", - type=click.Choice(["bz2", "gz", "xz"], case_sensitive=False), + type=click.Choice(["bz2", "gz", "xz", "none"], case_sensitive=False), help="Target compression format for on-the-fly conversion during download. " "Source compression is detected automatically from the file extension. " - "All compressed files will be converted to the target format (bz2, gz, xz).", + "Use 'none' to decompress files without recompressing.", ) @click.option( "--format", diff --git a/tests/test_compression_conversion.py b/tests/test_compression_conversion.py index 6e0c9a6..39aa069 100644 --- a/tests/test_compression_conversion.py +++ b/tests/test_compression_conversion.py @@ -198,6 +198,43 @@ def test_corrupted_file_handling(): # Verify target file was cleaned up assert not os.path.exists(target_file) +def test_should_convert_compression_none_on_compressed(): + """--compression none on a compressed file: should convert, source detected.""" + should_convert, source = _should_convert_compression("file.txt.bz2", "none") + assert should_convert is True + assert source == "bz2" + + +def test_should_convert_compression_none_on_uncompressed(): + """--compression none on an uncompressed file: nothing to do.""" + should_convert, source = _should_convert_compression("file.txt", "none") + assert should_convert is False + assert source is None + + +def test_get_converted_filename_none_strips_extension(): + """--compression none: strips compression extension, adds nothing.""" + assert _get_converted_filename("data.txt.bz2", "bz2", "none") == "data.txt" + assert _get_converted_filename("data.txt.gz", "gz", "none") == "data.txt" + assert _get_converted_filename("data.txt.xz", "xz", "none") == "data.txt" + + +def test_decompress_bz2_to_plain(): + """--compression none on bz2 file decompresses to plain file via _convert_compression_format.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_data = b"Decompression test data" * 50 + + bz2_file = os.path.join(tmpdir, "test.txt.bz2") + with bz2.open(bz2_file, "wb") as f: + f.write(test_data) + + plain_file = os.path.join(tmpdir, "test.txt") + _convert_compression_format(bz2_file, plain_file, "bz2", "none") + + assert not os.path.exists(bz2_file) + assert os.path.exists(plain_file) + with open(plain_file, "rb") as f: + assert f.read() == test_data if __name__ == "__main__": pytest.main([__file__, "-v"]) \ No newline at end of file