Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ docker run --rm -v $(pwd):/data dbpedia/databus-python-client download $DOWNLOAD
- `--databus-key`
- If the databus is protected and needs API key authentication, you can provide the API key with `--databus-key YOUR_API_KEY`.
- `--compression`
- Enables on-the-fly compression format conversion during download. Supported formats: `bz2`, `gz`, `xz`. The source compression is auto-detected from the file extension. Example: `--compression gz` converts all downloaded compressed files to gzip format.
- Enables on-the-fly compression format conversion during download. Supported formats: `bz2`, `gz`, `xz`, `none`. The source compression is auto-detected from the file extension. Use `none` to decompress files without recompressing. Example: `--compression gz` converts all downloaded compressed files to gzip format.
- `--format`
- Enables on-the-fly RDF and tabular format conversion during download (Layer 2 and Layer 3). Supported formats: `ntriples` (`nt`), `turtle` (`ttl`), `rdf-xml` (`rdf`, `xml`), `nquads` (`nq`), `trig`, `trix`, `json-ld` (`jsonld`), `csv`, `tsv`. Short aliases shown in brackets. Only the converted output file is kept — the original is deleted after successful conversion. Within the same equivalence class (e.g. turtle to ntriples) conversion is lossless. Across classes (e.g. RDF to CSV) some flags below may be required.
- `--graph-name`
Expand Down Expand Up @@ -284,6 +284,9 @@ databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased

# Download a collection and unify all files to bz2 format
databusclient download https://databus.dbpedia.org/dbpedia/collections/dbpedia-snapshot-2022-12 --compression bz2

# Decompress files without recompressing
databusclient download https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals/2022.12.01/mappingbased-literals_lang=az.ttl.bz2 --compression none
```

**Download with Format Conversion**: download files and convert RDF or tabular format on-the-fly. Only the converted output file is kept.
Expand Down
78 changes: 62 additions & 16 deletions databusclient/api/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,15 @@ def _should_convert_compression(
"""Determine if a file should have its compression format converted or compressed.

Source compression is detected automatically from the file extension.
If compression='none', compressed files are decompressed and saved without
any compression. If the file is already uncompressed and compression='none',
nothing is done.
If the file is uncompressed and a target compression is specified,
it will be compressed to the target format (source_format returned as None).

Args:
filename: Name of the file.
compression: Target compression format ('bz2', 'gz', 'xz') or None.
compression: Target compression format ('bz2', 'gz', 'xz', 'none') or None.

Returns:
Tuple of (should_convert: bool, source_format: Optional[str]).
Expand All @@ -80,7 +83,14 @@ def _should_convert_compression(

source_format = _detect_compression_format(filename)

# If file is not compressed, don't convert
# 'none' means decompress — only meaningful if file is compressed
if compression.lower() == "none":
if source_format is None:
# Already uncompressed, nothing to do
return False, None
return True, source_format

# If file is not compressed, compress it to the target format
if source_format is None:
return True, None

Expand All @@ -99,12 +109,21 @@ def _get_converted_filename(
Args:
filename: Original filename.
source_format: Source compression format ('bz2', 'gz', 'xz').
target_format: Target compression format ('bz2', 'gz', 'xz').
target_format: Target compression format ('bz2', 'gz', 'xz') or 'none'
to decompress without recompressing.

Returns:
New filename with updated extension.
New filename with updated extension. If target_format is 'none',
the compression extension is stripped and nothing is added.
"""
source_ext = COMPRESSION_EXTENSIONS[source_format]

# 'none' means decompress — strip compression extension, add nothing
if target_format.lower() == "none":
if filename.lower().endswith(source_ext):
return filename[: -len(source_ext)]
return filename

target_ext = COMPRESSION_EXTENSIONS[target_format]

# Handle case-insensitive extension matching
Expand All @@ -116,36 +135,57 @@ def _get_converted_filename(
def _convert_compression_format(
source_file: str, target_file: str, source_format: str, target_format: str
) -> None:
"""Convert a compressed file from one format to another.
"""Convert or decompress a compressed file.

Handles two cases:
- target_format is 'none': decompress source_file to target_file without recompressing.
- target_format is a compression format: decompress then recompress to target format.

Args:
source_file: Path to source compressed file.
target_file: Path to target compressed file.
target_file: Path to target file.
source_format: Source compression format ('bz2', 'gz', 'xz').
target_format: Target compression format ('bz2', 'gz', 'xz').
target_format: Target compression format ('bz2', 'gz', 'xz') or 'none' to decompress only.

Raises:
ValueError: If source_format or target_format is not supported.
RuntimeError: If compression conversion fails.
ValueError: If source_format is not supported.
RuntimeError: If the operation fails.
"""
# Validate compression formats
if source_format not in COMPRESSION_MODULES:
raise ValueError(
f"Unsupported source compression format: {source_format}. Supported formats: {list(COMPRESSION_MODULES.keys())}"
f"Unsupported source compression format: {source_format}. "
f"Supported formats: {list(COMPRESSION_MODULES.keys())}"
)

source_module = COMPRESSION_MODULES[source_format]

# Decompression-only path: target_format == 'none'
if target_format.lower() == "none":
print(f"Decompressing {os.path.basename(source_file)} -> {os.path.basename(target_file)}")
try:
with source_module.open(source_file, "rb") as sf:
with open(target_file, "wb") as tf:
shutil.copyfileobj(sf, tf)
os.remove(source_file)
print(f"Decompression complete: {os.path.basename(target_file)}")
except Exception as e:
if os.path.exists(target_file):
os.remove(target_file)
raise RuntimeError(f"Decompression failed: {e}")
return

if target_format not in COMPRESSION_MODULES:
raise ValueError(
f"Unsupported target compression format: {target_format}. Supported formats: {list(COMPRESSION_MODULES.keys())}"
f"Unsupported target compression format: {target_format}. "
f"Supported formats: {list(COMPRESSION_MODULES.keys())}"
)

source_module = COMPRESSION_MODULES[source_format]
target_module = COMPRESSION_MODULES[target_format]

print(
f"Converting {source_format} → {target_format}: {os.path.basename(source_file)}"
)

# Decompress and recompress with progress indication
chunk_size = 8192

try:
Expand Down Expand Up @@ -561,6 +601,11 @@ def _download_file(
shutil.copyfileobj(sf, tf)
os.remove(filename)
print(f"Compression complete: {os.path.basename(target_filepath)}")
elif compression.lower() == "none":
# Decompress — strip compression extension, save plain file.
target_filename = _get_converted_filename(file, source_fmt, "none")
target_filepath = os.path.join(localDir, target_filename)
_convert_compression_format(filename, target_filepath, source_fmt, "none")
else:
target_filename = _get_converted_filename(file, source_fmt, compression)
target_filepath = os.path.join(localDir, target_filename)
Expand Down Expand Up @@ -707,10 +752,11 @@ def _download_file(
# 4. Source was NOT compressed, no --compression given -> no compression
if source_compression is not None:
if should_convert_compression and compression:
final_compression = compression
# 'none' means no recompression after format conversion
final_compression = None if compression.lower() == "none" else compression
Comment thread
Integer-Ctrl marked this conversation as resolved.
else:
final_compression = source_compression
elif compression:
elif compression and compression.lower() != "none":
# Source was uncompressed but user explicitly requested --compression
final_compression = compression
else:
Expand Down
4 changes: 2 additions & 2 deletions databusclient/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,10 +251,10 @@ def _write_manifest():
@click.option(
"--compression",
"compression",
type=click.Choice(["bz2", "gz", "xz"], case_sensitive=False),
type=click.Choice(["bz2", "gz", "xz", "none"], case_sensitive=False),
help="Target compression format for on-the-fly conversion during download. "
"Source compression is detected automatically from the file extension. "
"All compressed files will be converted to the target format (bz2, gz, xz).",
"Use 'none' to decompress files without recompressing.",
)
@click.option(
"--format",
Expand Down
37 changes: 37 additions & 0 deletions tests/test_compression_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,43 @@ def test_corrupted_file_handling():
# Verify target file was cleaned up
assert not os.path.exists(target_file)

def test_should_convert_compression_none_on_compressed():
"""--compression none on a compressed file: should convert, source detected."""
should_convert, source = _should_convert_compression("file.txt.bz2", "none")
assert should_convert is True
assert source == "bz2"


def test_should_convert_compression_none_on_uncompressed():
"""--compression none on an uncompressed file: nothing to do."""
should_convert, source = _should_convert_compression("file.txt", "none")
assert should_convert is False
assert source is None


def test_get_converted_filename_none_strips_extension():
"""--compression none: strips compression extension, adds nothing."""
assert _get_converted_filename("data.txt.bz2", "bz2", "none") == "data.txt"
assert _get_converted_filename("data.txt.gz", "gz", "none") == "data.txt"
assert _get_converted_filename("data.txt.xz", "xz", "none") == "data.txt"


def test_decompress_bz2_to_plain():
"""--compression none on bz2 file decompresses to plain file via _convert_compression_format."""
with tempfile.TemporaryDirectory() as tmpdir:
test_data = b"Decompression test data" * 50

bz2_file = os.path.join(tmpdir, "test.txt.bz2")
with bz2.open(bz2_file, "wb") as f:
f.write(test_data)

plain_file = os.path.join(tmpdir, "test.txt")
_convert_compression_format(bz2_file, plain_file, "bz2", "none")

assert not os.path.exists(bz2_file)
assert os.path.exists(plain_file)
with open(plain_file, "rb") as f:
assert f.read() == test_data
Comment thread
Integer-Ctrl marked this conversation as resolved.

if __name__ == "__main__":
pytest.main([__file__, "-v"])
Loading