new features - #2
Draft
softlukas wants to merge 8 commits into
Draft
Conversation
softlukas
marked this pull request as draft
July 23, 2026 13:25
added 2 commits
July 23, 2026 18:12
…tion - Add idempotent install/migrate_db.py for Stage 1 schema - Add common/content_storage.py for SHA-256 based content deduplication - Extend etc/config.yaml with content_storage_path, max_storage_size_gb, sandbox_providers - Create /data/url_evaluator/content in prepare_environment.sh - Run migrate_db.py after create_db.sql in configure_sqlite.sh
- Implement SHA-256 based storage path layout. - Add save_content, load_content, content_exists helpers. - Add delete_content helper with empty parent directory cleanup. - Normalise and validate SHA-256 hashes in storage_path.
added 5 commits
July 24, 2026 10:56
- Add common/db_helpers.py with persist_content_snapshot(), set_url_latest_content() and record_url_history(). - Extend bin/evaluator.analyze_content() to save downloaded content, metadata and update content_snapshot/url_content/urls/url_history. - Keep existing VT/MB classification logic unchanged.
- Add record_url_source() and record_discovered_url() helpers to common/utils.py - Update process_new_session() to populate source_detail, origin_url, observed_at, idea_id and session_hash in url_source and discovered_at in discovered_urls - Update bin/honeynetasia2evaluator.py to record observed_at via record_url_source() - Update bin/tpot2evaluator.py to pass sensor/host as source_detail - Update bin/warden2evaluator.py to pass event metadata through process_new_session() - Ensure migrate_db.py creates unique index on url_source(url, source) for upserts
- Add global search box and bulk URL textarea to list page - Show observations, content history, HTTP headers, change history on detail page - Add download blob and request-sandbox stub routes - Make MISP optional for dev/local runs
- evaluator2misp.py / evaluator2misp_bysource.py: - add latest_content_hash (sha256) and content_size to exported MISP objects - keep existing sha1 hash and source metadata - switch source aggregation to SQLite-compatible subquery to avoid DISTINCT aggregate errors - evaluator2urlhaus.py: - include sha256, sha1, file_mime_type and content_size in URLhaus submissions - still only send active malicious URLs, skipping already-blacklisted ones
| @@ -100,9 +147,16 @@ def process_new_session(db, config, session, idea_id, detect_time, source, sourc | |||
| ) | |||
| for url, occurrences in Counter(extracted_urls).items(): | |||
| db.execute("INSERT OR IGNORE INTO url_session (url, session) VALUES (?, ?)", (url, session_hash)) | |||
Comment on lines
+77
to
+80
| """ | ||
| Record an observation of `url` in `url_source`, preserving first-seen time | ||
| and updating the last-seen time on duplicate observations. | ||
| """ |
Comment on lines
+361
to
+365
| try: | ||
| sessions = db.execute("SELECT sessions.session, sessions.idea_id FROM sessions JOIN url_session ON url_session.session=sessions.session_hash WHERE url_session.url = ?", (url,)).fetchall() | ||
| except Exception: | ||
| # url_session may not exist in migrated databases yet | ||
| sessions = [] |
| requested_by TEXT | ||
| ); | ||
|
|
||
| CREATE INDEX idx_url_source_lookup ON url_source(url, source, observed_at); |
Comment on lines
+142
to
+166
| if is_new: | ||
| db.execute( | ||
| """ | ||
| INSERT OR IGNORE INTO content_snapshot | ||
| (content_hash, url, downloaded_at, source_ip, server_ip, http_status, | ||
| http_headers, mime_type, content_size, storage_path, sha1, sha256) | ||
| VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) | ||
| """, | ||
| ( | ||
| sha256, | ||
| url, | ||
| downloaded_at, | ||
| source_ip, | ||
| server_ip, | ||
| http_status, | ||
| http_headers, | ||
| mime_type, | ||
| content_size, | ||
| storage_path_rel, | ||
| sha1, | ||
| sha256, | ||
| ), | ||
| ) | ||
| logger.debug(f"Recorded content snapshot {sha256} for {url}") | ||
|
|
| FROM url_content uc | ||
| JOIN content_snapshot cs ON cs.content_hash = uc.content_hash | ||
| WHERE uc.url = ? | ||
| ORDER BY uc.first_seen DESC |
Comment on lines
+529
to
+533
| snapshot = db.execute( | ||
| "SELECT 1 FROM content_snapshot WHERE content_hash = ?", (content_hash,) | ||
| ).fetchone() | ||
| if not snapshot: | ||
| return make_response(jsonify({'error': 'Content snapshot not found'}), 404) |
| from common.db import SQLiteWrapper | ||
| from common.db_helpers import update_url_field | ||
| from common.utils import is_valid, get_domain | ||
| from common.content_storage import load_content, storage_path |
| </td> | ||
| <td> | ||
| {% if obs.idea_id %} | ||
| <a target="_blank" href="https://mentat-hub.cesnet.cz/mentat/events/{{ obs.idea_id }}/show" title="Event detail"><img class="mentat" src="{{ url_for("static", filename="img/mentat.png") }}"></a> |
Comment on lines
+261
to
+263
| <td class="hash-cell"><code>{{ entry.sha1[:16] if entry.sha1 else '-' }}...</code></td> | ||
| <td>{{ entry.mime_type or '-' }}</td> | ||
| <td>{{ entry.content_size if entry.content_size is not none else '-' }} B</td> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
No description provided.