From 3c374fc1e2835158a9b2473e71e83b570274b9e3 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 3 Dec 2024 15:25:14 -0700 Subject: [PATCH] updates --- README.md | 2 +- src/maltest/tests/test_common_protocols.py | 119 ++++++++++++++++++ src/maltest/tests/test_malcolm_pcap.py | 103 --------------- ...snpp_protocols.py => test_ot_protocols.py} | 9 +- 4 files changed, 127 insertions(+), 106 deletions(-) create mode 100644 src/maltest/tests/test_common_protocols.py delete mode 100644 src/maltest/tests/test_malcolm_pcap.py rename src/maltest/tests/{test_icsnpp_protocols.py => test_ot_protocols.py} (97%) diff --git a/README.md b/README.md index ca13b51..e306830 100644 --- a/README.md +++ b/README.md @@ -299,4 +299,4 @@ See the following tests for examples of how to access and use these fixtures: * [test_malcolm_response.py](src/maltest/tests/test_malcolm_response.py) - querying the [Malcolm API](https://idaholab.github.io/Malcolm/docs/api.html#API) using the [Requests](https://requests.readthedocs.io/en/latest/) library * [test_malcolm_db_health.py](src/maltest/tests/test_malcolm_db_health.py) - querying the [data store](https://idaholab.github.io/Malcolm/docs/opensearch-instances.html#OpenSearchInstance) directly using the [OpenSearch](https://opensearch.org/docs/latest/clients/python-low-level/) or [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/index.html) client -* [test_icsnpp_protocols.py](src/maltest/tests/test_icsnpp_protocols.py) - querying the [Malcolm Field Aggregation API](https://idaholab.github.io/Malcolm/docs/api-aggregations.html), specifying a `from` query start time filter to search all historical data, a filter on `event.provider` to limit the result set to records from Zeek, and a `tags` filter to limit the matching records to the tags represented by the uploaded PCAPs (see above) +* [test_common_protocols.py](src/maltest/tests/test_common_protocols.py) - querying the [Malcolm Field Aggregation API](https://idaholab.github.io/Malcolm/docs/api-aggregations.html), specifying a `from` query start time filter to search all historical data, a filter on `event.provider` to limit the result set to records from Zeek, and a `tags` filter to limit the matching records to the tags represented by the uploaded PCAPs (see above) diff --git a/src/maltest/tests/test_common_protocols.py b/src/maltest/tests/test_common_protocols.py new file mode 100644 index 0000000..4cd76fc --- /dev/null +++ b/src/maltest/tests/test_common_protocols.py @@ -0,0 +1,119 @@ +import mmguero +import requests +import logging + +LOGGER = logging.getLogger(__name__) + +UPLOAD_ARTIFACTS = [ + "DCERPC.pcap", + "DHCP.pcap", + "DNS.pcap", + "FTP.pcap", + "HTTP_1.pcap", + "HTTP_2.pcap", + "IPsec.pcap", + "IRC.pcap", + "KRB5.pcap", + "LDAP.pcap", + "MySQL.pcap", + "NTLM.pcap", + "NTP.pcap", + "OpenVPN.pcap", + "OSPF.pcap", + "QUIC.pcap", + "RADIUS.pcap", + "RDP.pcap", + "RFB.pcap", + "SIP.pcap", + "SMB.pcap", + "SMTP.pcap", + "SNMP.pcap", + "SSH.pcap", + "SSL.pcap", + "STUN.pcap", + "Syslog.pcap", + "Telnet.pcap", + "TFTP.pcap", + "Tunnels.pcap", + "WireGuard.pcap", +] + +EXPECTED_DATASETS = [ + "conn", + "dce_rpc", + "dhcp", + "dns", + "dpd", + "files", + "ftp", + "gquic", + "http", + "ipsec", + "irc", + "ja4ssh", + "kerberos", + "known_certs", + "known_hosts", + "known_services", + "ldap", + "ldap_search", + "login", + "mysql", + "notice", + "ntlm", + "ntp", + "ocsp", + "ospf", + "pe", + "radius", + "rdp", + "rfb", + "sip", + "smb_cmd", + "smb_files", + "smb_mapping", + "smtp", + "snmp", + "socks", + "software", + "ssh", + "ssl", + "stun", + "stun_nat", + "syslog", + "tftp", + "tunnel", + "websocket", + "weird", + "wireguard", + "x509", +] + + +def test_common_protocols( + malcolm_http_auth, + malcolm_url, + pcap_hash_map, +): + assert all([pcap_hash_map.get(x, None) for x in mmguero.GetIterable(UPLOAD_ARTIFACTS)]) + + response = requests.post( + f"{malcolm_url}/mapi/agg/event.dataset", + headers={"Content-Type": "application/json"}, + json={ + "from": "0", + "filter": { + "event.provider": "zeek", + "tags": [pcap_hash_map[x] for x in mmguero.GetIterable(UPLOAD_ARTIFACTS)], + }, + }, + allow_redirects=True, + auth=malcolm_http_auth, + verify=False, + ) + response.raise_for_status() + buckets = { + item['key']: item['doc_count'] for item in mmguero.DeepGet(response.json(), ['event.dataset', 'buckets'], []) + } + LOGGER.info(buckets) + assert all([(buckets.get(x, 0) > 0) for x in EXPECTED_DATASETS]) diff --git a/src/maltest/tests/test_malcolm_pcap.py b/src/maltest/tests/test_malcolm_pcap.py deleted file mode 100644 index 24c0a1b..0000000 --- a/src/maltest/tests/test_malcolm_pcap.py +++ /dev/null @@ -1,103 +0,0 @@ -import mmguero - -UPLOAD_ARTIFACTS = [ - "other/malcolm-micro-ctf.pcap", - "plugins/cve-2020-0601/broken.pcap", - "plugins/cve-2020-0601/ecdsa-cert.pcap", - "plugins/cve-2020-0601/explicit.pcap", - "plugins/cve-2020-0601/exploit.pcap", - "plugins/cve-2020-13777/chrome-34-google.trace", - "plugins/cve-2020-13777/gnutls-tls1.2-non-vulnerable.pcap", - "plugins/cve-2020-13777/gnutls-tls1.2-vulnerable.pcap", - "plugins/cve-2020-13777/gnutls-tls1.3.pcap", - "plugins/CVE-2020-16898/6in4-linklocal-hlimit-less255.pcap", - "plugins/CVE-2020-16898/ipv6-neighbor-discovery.pcap", - "plugins/CVE-2020-16898/ipv6-router-advertisement-leaving.pcap", - "plugins/CVE-2020-16898/pi3_poc.pcap", - "plugins/CVE-2020-16898/RS-RA.pcap", - "plugins/CVE-2021-38647/CVE-2021-38647-exploit-craigmunsw-omigod-lab.pcap", - "plugins/CVE-2021-41773/apache_exploit_success.pcap", - "plugins/cve-2021-44228/2021-12-11-thru-13-server-activity-with-log4j-attempts.pcap", - "plugins/cve-2021-44228/log4j-attack.pcap", - "plugins/cve-2021-44228/log4j-dns_exfil.pcap", - "plugins/cve-2021-44228/log4j-user_agent.pcap", - "plugins/cve-2021-44228/log4j-webapp.pcap", - "plugins/cve-2021-44228/spcap-CEXKLs3NQWdEM2CoMj-1639421287179170294-1.pcap", - "plugins/cve-2022-26809/cve-2022-26809-4.pcap", - "plugins/CVE-2022-3602/sample_OpenSSLv3.0.5.pcap", - "plugins/CVE-2022-3602/spookyssl-merged.pcap", - "plugins/download_over_dns.pcap", - "plugins/http-more-files-names/favicon.pcap", - "plugins/http-more-files-names/http-etag-and-filename.pcap", - "plugins/http-more-files-names/http-filename-and-etag.pcap", - "plugins/http-more-files-names/http-filename.pcap", - "plugins/quasarrat.pcap", - "plugins/smb_mimikatz_copy_to_host.pcap", - "plugins/zeek-EternalSafety/doublepulsar-backdoor-connect-win7.pcap", - "plugins/zeek-EternalSafety/esteemedaudit-failed-XPSP2.pcap", - "plugins/zeek-EternalSafety/eternalblue-failed-patched-win7.pcap", - "plugins/zeek-EternalSafety/eternalblue-success-unpatched-win7.pcap", - "plugins/zeek-EternalSafety/eternalchampion.pcap", - "plugins/zeek-EternalSafety/eternalromance-doublepulsar-meterpreter.pcap", - "plugins/zeek-EternalSafety/eternalromance-success-2008r2.pcap", - "plugins/zeek-EternalSafety/metasploit-ms017-010-win7x64.pcap", - "plugins/zeek-EternalSafety/wannacry.pcap", - "plugins/zeek-httpattacks/http.trace", - "plugins/zeek-sniffpass/http_post.trace", - "plugins/zeek-xor-exe-plugin/2015-04-09-Nuclear-EK-traffic.pcap", - "plugins/zerologon/CVE-2020-1472_exploit_win2016.pcap", - "plugins/zerologon/CVE-2020-1472_exploit_win2019.pcap", - "plugins/zerologon/CVE-2020-1472_test_win2016.pcap", - "plugins/zerologon/CVE-2020-1472_test_win2019.pcap", - "protocols/BACnet.pcap", - "protocols/BSAP.pcap", - "protocols/DCERPC.pcap", - "protocols/DHCP.pcap", - "protocols/DNP3.pcap", - "protocols/DNS.pcap", - "protocols/ENIP.pcap", - "protocols/ETHERCAT.pcap", - "protocols/FTP.pcap", - "protocols/GENISYS.pcap", - "protocols/HARTIP.pcap", - "protocols/HTTP_1.pcap", - "protocols/HTTP_2.pcap", - "protocols/IPsec.pcap", - "protocols/IRC.pcap", - "protocols/KRB5.pcap", - "protocols/LDAP.pcap", - "protocols/Modbus.pcap", - "protocols/MQTT.pcap", - "protocols/MySQL.pcap", - "protocols/NTLM.pcap", - "protocols/NTP.pcap", - "protocols/OPCUA-Binary.pcap", - "protocols/OpenVPN.pcap", - "protocols/OSPF.pcap", - "protocols/PROFINET.pcap", - "protocols/QUIC.pcap", - "protocols/RADIUS.pcap", - "protocols/RDP.pcap", - "protocols/RFB.pcap", - "protocols/S7comm.pcap", - "protocols/SIP.pcap", - "protocols/SMB.pcap", - "protocols/SMTP.pcap", - "protocols/SNMP.pcap", - "protocols/SSH.pcap", - "protocols/SSL.pcap", - "protocols/STUN.pcap", - "protocols/Synchrophasor.pcap", - "protocols/Syslog.pcap", - "protocols/TDS.pcap", - "protocols/Telnet.pcap", - "protocols/TFTP.pcap", - "protocols/Tunnels.pcap", - "protocols/WireGuard.pcap", -] - - -def test_malcolm_pcap_hash( - pcap_hash_map, -): - assert all([pcap_hash_map.get(x, None) for x in mmguero.GetIterable(UPLOAD_ARTIFACTS)]) diff --git a/src/maltest/tests/test_icsnpp_protocols.py b/src/maltest/tests/test_ot_protocols.py similarity index 97% rename from src/maltest/tests/test_icsnpp_protocols.py rename to src/maltest/tests/test_ot_protocols.py index 4b2a95a..eea1825 100644 --- a/src/maltest/tests/test_icsnpp_protocols.py +++ b/src/maltest/tests/test_ot_protocols.py @@ -21,7 +21,12 @@ "protocols/TDS.pcap", ] -# TODO: genisys is not enabled by default? +# TODO: +# "ecat_arp_info", +# "ecat_foe_info", +# "ecat_soe_info", +# "ge_srtp", +# "genisys", EXPECTED_DATASETS = [ "bacnet", "bacnet_device_control", @@ -109,7 +114,7 @@ ] -def test_icsnpp_protocols( +def test_ot_protocols( malcolm_http_auth, malcolm_url, pcap_hash_map,