WIP - Not ready for Merge - Consolidate spread Pi.Alert work

README.md

@@ -1,6 +1,7 @@
 # Pi.Alert
 <!--- --------------------------------------------------------------------- --->
 
+
 WIFI / LAN intruder detector.
 
 Scan the devices connected to your WIFI / LAN and alert you the connection of
@@ -21,14 +22,16 @@ The system continuously scans the network for:
   - Internet IP address changes
 
 ## Scan Methods
-Up to three scanning methods are used:
+Up to four scanning methods are used:
   - **Method 1: arp-scan**. The arp-scan system utility is used to search
         for devices on the network using arp frames.
-  - **Method 2: Pi-hole**. This method is optional and complementary to
+  - **Method 2: nmap**. The nmap system utility is used to search for
+        devices on the network using ping frames.
+  - **Method 3: Pi-hole**. This method is optional and complementary to
         method 1. If the Pi-hole DNS server is active, Pi.Alert examines its
         activity looking for active devices using DNS that have not been
         detected by method 1.
-  - **Method 3. dnsmasq**. This method is optional and complementary to the
+  - **Method 4. dnsmasq**. This method is optional and complementary to the
         previous methods. If the DHCP server dnsmasq is active, Pi.Alert
         examines the DHCP leases (addresses assigned) to find active devices
         that were not discovered by the other methods.
@@ -110,6 +113,7 @@ Linux distributions.
   | arp-scan     | Scan network using arp commands        |
   | Pi.hole      | DNS Server with Ad-block               |
   | dnsmasq      | DHCP Server                            |
+  | nmap         | Network Mapper                         |
 
 ### License
   GPL 3.0

back/pialert.py

@@ -27,6 +27,10 @@
 import io
 import smtplib
 import csv
+import json
+import requests
+import xml.etree.ElementTree as ET
+
 
 
 #===============================================================================
@@ -354,6 +358,7 @@ def scan_network ():
     # TESTING - Fast scan
         # arpscan_retries = 1
 
+    # ScanCycle data        
     # arp-scan command
     print ('\nScanning...')
     print ('    arp-scan Method...')
@@ -363,6 +368,14 @@ def scan_network ():
     # DEBUG - print number of rows updated
         # print (arpscan_devices)
 
+    # nmap command
+    print ('    nmap Method...')
+    print_log ('nmap starts...')
+    arpscan_devices = execute_nmap (NMAP_OPTIONS, arpscan_devices)
+    print_log ('nmap ends')
+    # DEBUG - print number of rows updated
+        # print (arpscan_devices)
+
     # Pi-hole method
     print ('    Pi-hole Method...')
     openDB()
@@ -480,17 +493,66 @@ def execute_arpscan (pRetries):
     unique_devices = [] 
 
     for device in devices_list :
+        device['mac'] = device['mac'].upper()
         if device['mac'] not in unique_mac: 
             unique_mac.append(device['mac'])
             unique_devices.append(device)
 
     # DEBUG
-        # print (devices_list)
-        # print (unique_mac)
-        # print (unique_devices)
-        # print (len(devices_list))
-        # print (len(unique_mac))
-        # print (len(unique_devices))
+    #    print (devices_list)
+    #    print (unique_mac)
+    #    print (unique_devices)
+    #    print (len(devices_list))
+    #    print (len(unique_mac))
+    #    print (len(unique_devices))
+
+    # return list
+    return unique_devices
+
+#-------------------------------------------------------------------------------
+def execute_nmap (NMAP_OPTIONS, arpscan_results):
+
+    # nmap subnet configuration
+    # Prepare command arguments - find local networks if no options passed
+    if not NMAP_OPTIONS:
+      local_net_cmd = ["ip addr|grep 'inet ' |grep -v 'lo$' | awk {'print $2'}"]
+      local_net = subprocess.Popen (local_net_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().strip().splitlines()
+      nmap_args=local_net
+    else:
+      nmap_args = NMAP_OPTIONS.split()
+    nmap_args = ['nmap', '-sn', '--send-ip', '-oX', '-' ] + nmap_args
+
+    # Execute command
+    nmap_output = subprocess.check_output (nmap_args, universal_newlines=True)
+    root = ET.fromstring(nmap_output);
+
+    # Track macs from arpscan to remove duplicates
+    unique_mac = [] 
+    unique_devices = [] 
+    for result in arpscan_results:
+      unique_mac.append(result['mac'])
+      unique_devices.append(result)
+
+    for child in root.iter('host'):
+      device = {}
+      for addr in child.iter('address'):
+        if addr.get('addrtype') == 'mac':
+            device['mac'] = addr.get('addr')
+            device['hw'] = addr.get('vendor')
+        if addr.get('addrtype') == 'ipv4':
+            device['ip'] = addr.get('addr')
+      if 'mac' in device:
+        device['mac'] = device['mac'].upper()
+        if re.match("[0-9A-F]{2}([-:]?)[0-9A-F]{2}(\\1[0-9A-F]{2}){4}$", device['mac']):
+          if device['mac'] not in unique_mac:
+            unique_mac.append(device['mac'])
+            unique_devices.append(device)
+
+    # DEBUG
+    #print (unique_mac)
+    #print (unique_devices)
+    #print (len(unique_mac))
+    #print (len(unique_devices))
 
     # return list
     return unique_devices
@@ -512,10 +574,11 @@ def copy_pihole_network ():
                         (SELECT name FROM PH.network_addresses
                          WHERE network_id = id ORDER BY lastseen DESC, ip),
                         (SELECT ip FROM PH.network_addresses
-                         WHERE network_id = id ORDER BY lastseen DESC, ip)
+                         WHERE network_id = id ORDER BY lastseen DESC, ip) as ip
                     FROM PH.network
                     WHERE hwaddr NOT LIKE 'ip-%'
-                      AND hwaddr <> '00:00:00:00:00:00' """)
+                      AND hwaddr <> '00:00:00:00:00:00'
+                      AND IP NOT LIKE 'fe%' """)
     sql.execute ("""UPDATE PiHole_Network SET PH_Name = '(unknown)'
                     WHERE PH_Name IS NULL OR PH_Name = '' """)
     # DEBUG
@@ -589,14 +652,14 @@ def save_scanned_devices (p_arpscan_devices, p_cycle_interval):
       # BUGFIX #106 - Device that pialert is running
         # local_mac_cmd = ["bash -lc ifconfig `ip route list default | awk {'print $5'}` | grep ether | awk '{print $2}'"]
           # local_mac_cmd = ["/sbin/ifconfig `ip route list default | sort -nk11 | head -1 | awk {'print $5'}` | grep ether | awk '{print $2}'"]
-    local_mac_cmd = ["/sbin/ifconfig `ip -o route get 1 | sed 's/^.*dev \\([^ ]*\\).*$/\\1/;q'` | grep ether | awk '{print $2}'"]
+    local_mac_cmd = ["/sbin/ip addr show dev `/sbin/ip -o route get 1 | sed -e 's/^.*dev //' -e 's/ .*$//'` | grep ether | awk '{print $2}'"]
     local_mac = subprocess.Popen (local_mac_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().strip()
 
     # local_dev_cmd = ["ip -o route get 1 | sed 's/^.*dev \\([^ ]*\\).*$/\\1/;q'"]
     # local_dev = subprocess.Popen (local_dev_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().strip()
 
     # local_ip_cmd = ["ip route list default | awk {'print $7'}"]
-    local_ip_cmd = ["ip -o route get 1 | sed 's/^.*src \\([^ ]*\\).*$/\\1/;q'"]
+    local_ip_cmd = ["/sbin/ip -o route get 1 | sed -e 's/^.*src //'  -e 's/ .*$//'"]
     local_ip = subprocess.Popen (local_ip_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().strip()
 
     # Check if local mac has been detected with other methods
@@ -617,7 +680,7 @@ def print_scan_stats ():
     sql.execute ("""SELECT COUNT(*) FROM CurrentScan
                     WHERE cur_ScanMethod='arp-scan' AND cur_ScanCycle = ? """,
                     (cycle,))
-    print ('        arp-scan Method....:', str (sql.fetchone()[0]) )
+    print ('        arp-scan/nmap Method....:', str (sql.fetchone()[0]) )
 
     # Devices Pi-hole
     sql.execute ("""SELECT COUNT(*) FROM CurrentScan
@@ -1331,6 +1394,16 @@ def email_reporting ():
             send_email (mail_text, mail_html)
         else :
             print ('    Skip mail...')
+        if REPORT_WEBHOOK :
+            print ('    Sending report by webhook...')
+            send_webhook (mail_text)
+        else :
+            print ('    Skip webhook...')
+        if REPORT_NTFY :
+            print ('    Sending report by NTFY...')
+            send_ntfy (mail_text)
+        else :
+            print ('    Skip NTFY...')
     else :
         print ('    No changes to report...')
 
@@ -1426,7 +1499,37 @@ def send_email (pText, pHTML):
     smtp_connection.sendmail (REPORT_FROM, REPORT_TO, msg.as_string())
     smtp_connection.quit()
 
+#-------------------------------------------------------------------------------
+def send_webhook (_Text):
+    #Define slack-compatible payload
+    _json_payload={
+    "username": "Pi.Alert",
+    "text": "There are new notifications",
+    "attachments": [{
+      "title": "Pi.Alert Notification",
+      "title_link": REPORT_DASHBOARD_URL,
+      "text": _Text
+    }]
+    }
+
+    # Using the Slack-Compatible Webhook endpoint for Discord so that the same payload can be used for both
+    if(WEBHOOK_URL.startswith('https://discord.com/api/webhooks/') and not WEBHOOK_URL.endswith("/slack")):
+        _WEBHOOK_URL = '{}/slack'.format(WEBHOOK_URL)
+    else:
+        _WEBHOOK_URL = WEBHOOK_URL
 
+    requests.post(_WEBHOOK_URL, json.dumps(_json_payload))
+#-------------------------------------------------------------------------------
+def send_ntfy (_Text):
+    requests.post("https://ntfy.sh/{}".format(NTFY_TOPIC),
+    data=_Text,
+    headers={
+        "Title": "Pi.Alert Notification",
+        "Click": REPORT_DASHBOARD_URL,
+        "Priority": "urgent",
+        "Tags": "warning"
+    })
+
 #===============================================================================
 # DB
 #===============================================================================

back/update_vendors.sh

@@ -24,17 +24,17 @@ sudo mkdir -p 2_backup
 sudo cp *.txt 2_backup
 sudo cp *.csv 2_backup
 
-sudo curl $1 -# -O http://standards-oui.ieee.org/iab/iab.csv
-sudo curl $1 -# -O http://standards-oui.ieee.org/iab/iab.txt
+sudo curl -L $1 -# -O http://standards-oui.ieee.org/iab/iab.csv
+sudo curl -L $1 -# -O http://standards-oui.ieee.org/iab/iab.txt
 
-sudo curl $1 -# -O http://standards-oui.ieee.org/oui28/mam.csv
-sudo curl $1 -# -O http://standards-oui.ieee.org/oui28/mam.txt
+sudo curl -L $1 -# -O http://standards-oui.ieee.org/oui28/mam.csv
+sudo curl -L $1 -# -O http://standards-oui.ieee.org/oui28/mam.txt
 
-sudo curl $1 -# -O http://standards-oui.ieee.org/oui36/oui36.csv
-sudo curl $1 -# -O http://standards-oui.ieee.org/oui36/oui36.txt
+sudo curl -L $1 -# -O http://standards-oui.ieee.org/oui36/oui36.csv
+sudo curl -L $1 -# -O http://standards-oui.ieee.org/oui36/oui36.txt
 
-sudo curl $1 -# -O http://standards-oui.ieee.org/oui/oui.csv
-sudo curl $1 -# -O http://standards-oui.ieee.org/oui/oui.txt
+sudo curl -L $1 -# -O http://standards-oui.ieee.org/oui/oui.csv
+sudo curl -L $1 -# -O http://standards-oui.ieee.org/oui/oui.txt
 
 
 # ----------------------------------------------------------------------

config/pialert.conf

@@ -7,35 +7,54 @@
 #  Puche 2021        [email protected]        GNU GPLv3
 #-------------------------------------------------------------------------------
 
+# General Pi.Alert settings
 PIALERT_PATH      = '/home/pi/pialert'
 DB_PATH           = PIALERT_PATH + '/db/pialert.db'
 LOG_PATH          = PIALERT_PATH + '/log'
 VENDORS_DB        = '/usr/share/arp-scan/ieee-oui.txt'
 PRINT_LOG         = False
+# QUERY_MYIP_SERVER = 'https://diagnostic.opendns.com/myip'
+QUERY_MYIP_SERVER = 'http://ipv4.icanhazip.com'
+
+# PiHole settings
+PIHOLE_ACTIVE     = False
+PIHOLE_DB         = '/etc/pihole/pihole-FTL.db'
+DHCP_ACTIVE       = False
+DHCP_LEASES       = '/etc/pihole/dhcp.leases'
+
+# DDNS settings
+DDNS_ACTIVE       = False
+DDNS_DOMAIN       = 'your_domain.freeddns.org'
+DDNS_USER         = 'dynu_user'
+DDNS_PASSWORD     = 'A0000000B0000000C0000000D0000000'
+DDNS_UPDATE_URL   = 'https://api.dynu.com/nic/update?'
 
+# Email SMTP settings
 SMTP_SERVER       = 'smtp.gmail.com'
 SMTP_PORT         = 587
 SMTP_USER         = '[email protected]'
 SMTP_PASS         = 'password'
 
+# Notification settings
 REPORT_MAIL       = False
+REPORT_WEBHOOK    = False
+REPORT_NTFY       = False
+REPORT_DASHBOARD_URL = 'http://pi.alert/'
+
+# Email Notification Settings
 REPORT_FROM       = 'Pi.Alert <' + SMTP_USER +'>'
 REPORT_TO         = '[email protected]'
 REPORT_DEVICE_URL = 'http://pi.alert/deviceDetails.php?mac='
 
-# QUERY_MYIP_SERVER = 'https://diagnostic.opendns.com/myip'
-QUERY_MYIP_SERVER = 'http://ipv4.icanhazip.com'
-DDNS_ACTIVE       = False
-DDNS_DOMAIN       = 'your_domain.freeddns.org'
-DDNS_USER         = 'dynu_user'
-DDNS_PASSWORD     = 'A0000000B0000000C0000000D0000000'
-DDNS_UPDATE_URL   = 'https://api.dynu.com/nic/update?'
+# Webhook Settings
+# if Discord: Add /slack after the url
+WEBHOOK_URL       = 'https://discord.com/api/webhooks/WEBHOOK_KEY/slack'
+WEBHOOK_FROM      = REPORT_FROM
 
-PIHOLE_ACTIVE     = False
-PIHOLE_DB         = '/etc/pihole/pihole-FTL.db'
-DHCP_ACTIVE       = False
-DHCP_LEASES       = '/etc/pihole/dhcp.leases'
+# Ntfy Settings
+NTFY_TOPIC        = 'replace_my_secure_topicname_91h889f28'
 
+# Scan settings
 # arp-scan options & samples
 #
 # Scan local network (default)
@@ -46,5 +65,21 @@ DHCP_LEASES       = '/etc/pihole/dhcp.leases'
 #
 # Scan using interface eth0
 # SCAN_SUBNETS    = '--localnet --interface=eth0'
+#
+# Scan from Docker
+# SCAN_SUBNETS    = '192.168.11.0/24 --interface=br0'
 
 SCAN_SUBNETS      = '--localnet'
+
+# nmap options & samples
+#
+# Scan local networks (default)
+# NMAP_OPTIONS      = ''
+#
+# Scan single subnet
+# NMAP_OPTIONS      = '192.168.11.0/24'
+#
+# Scan two subnets
+# NMAP_OPTIONS      = '192.168.11.0/24 192.168.144.0/24'
+
+NMAP_OPTIONS      = ''

config/version.conf

@@ -1,3 +1,3 @@
-VERSION           = '3.02'
-VERSION_YEAR      = '2021'
-VERSION_DATE      = '2021-04-24'
+VERSION           = '3.1'
+VERSION_YEAR      = '2022'
+VERSION_DATE      = '2022-03-28'

docs/INSTALL.md

@@ -226,7 +226,7 @@ block is not necessary
   requests to the correct pialert web folder
   ```
   sudo cp ~/pialert/install/pialert_front.conf /etc/lighttpd/conf-available
-  sudo ln -s ../conf-available/pialert_front.conf /etc/lighttpd/conf-enabled/pialert_front.conf
+  sudo ln -s /etc/lighttpd/conf-available/pialert_front.conf /etc/lighttpd/conf-enabled/pialert_front.conf
   sudo /etc/init.d/lighttpd restart
   ```
 

front/php/server/devices.php

@@ -422,9 +422,9 @@ function getDeviceTypes() {
           WHERE dev_DeviceType NOT IN ("",
                  "Smartphone", "Tablet",
                  "Laptop", "Mini PC", "PC", "Printer", "Server", "Singleboard Computer (SBC)",
-                 "Game Console", "SmartTV", "TV Decoder", "Virtual Assistance",
+                 "Domotic", "Game Console", "SmartTV", "TV Decoder", "Virtual Assistance",
                  "Clock", "House Appliance", "Phone", "Radio",
-                 "AP", "NAS", "PLC", "Router")
+                 "AP", "NAS", "PLC", "Router", "USB LAN Adapter", "USB WIFI Adapter", "Other")
 
           UNION SELECT 1 as dev_Order, "Smartphone"
           UNION SELECT 1 as dev_Order, "Tablet"

install/pialert_front.conf

@@ -7,6 +7,6 @@
 #  Puche 2021        [email protected]        GNU GPLv3
 # ------------------------------------------------------------------------------
 
-$HTTP["host"] == "pi.alert" {
+$HTTP["host"] =~ "pi\.alert" {
   server.document-root = "/var/www/html/pialert/"
 }

install/pialert_install.sh

@@ -143,7 +143,7 @@ ask_config() {
 
   if $DHCP_ACTIVATE ; then
     msgbox "Default DHCP options will be used. Range=$DHCP_RANGE_START - $DHCP_RANGE_END / Router=$DHCP_ROUTER / Domain=$DHCP_DOMAIN / Leases=$DHCP_LEASE h." \
-           "Yo can change this values in your Pi-hole Admin Portal"
+           "You can change this values in your Pi-hole Admin Portal"
     msgbox "Make sure your router's DHCP server is disabled" \
            "when using the Pi-hole DHCP server!"
   fi