IEEE Ethernet MAC address registry changes (#79)

* Changed get-oui to fetch and concat all IEEE registry data. Obsoleted get-iab

* Removed references to ieee-iab. Updated manpages. New ieee-oui file.

* Updated ChangeLog

* IEEE entries now 42874 that's 85% of 50K limit. Raise HASH_TABLE_SIZE to 60K

* Substitute PKGDATADIR in get-oui manpage

* Correct make distcheck failure

* Added check for IEEE registry lookup

* Changed IEEE URLs to HTTPS. Manpage updates. Updated ieee-oui.txt

* Minor changes. Updated ChangeLog

ChangeLog

@@ -1,22 +1,54 @@
-2021-10-30 Roy Hills <royhills@hotmail.com>
+2022-11-01 Roy Hills <royhills@hotmail.com>
+
+	* arp-scan.h: Increase HASH_TABLE_SIZE from 50,000 to 70,000 due to
+	  addition of IEEE MA-M and MA-S registries.  ieee-oui.txt currently
+	  contains 47028 entries of 70000 (67.2%).
+
+	* get-oui.1: renamed to get-oui.1.dist with PKGDATADIR substitution.
+
+	* check-ieee-reg: New file to test IEEE registry lookup.
+
+	* Makefile.am: Build man page "get-oui.1" from "get-oui.dist", added
+	  check-ieee-reg test.
+
+	* get-oui: Changed IEEE URLs from http to https. This avoids an HTTP
+	  redirect from the http to https site.
+
+	* ieee-oui.txt: Updated from IEEE website.
+
+2022-10-31 Roy Hills <royhills@hotmail.com>
+
+	* get-oui: Changed to fetch registry data from all four IEEE registries
+	  (MA-L, MA-M, MA-S and IAB), format appropriately for arp-scan, and
+	  concatenate into ieee-oui.
+	* get-iab: No longer needed. Replaced with stub script that calls die.
+	* Makefile.am, check-decode, check-host-list: Removed get-iab / iabfile
+	  references.
+	* arp-scan.c, arp-scan.h: Removed --iabfile option.
+	* README.md, arp-scan.1.dist, get-oui.1, mac-vendor.5, mac-vendor.txt:
+	  Update text to reflect changes.
+	* get-iab.1: Removed. No longer needed.
+	* ieee-iab.txt: Removed. No longer needed.
+	* ieee-oui.txt: Updated to new combined registry format.
+
+2022-10-30 Roy Hills <royhills@hotmail.com>
 
 	* Makefile.am: Build man page "arp-scan.1" from "arp-scan.dist"
 	  replacing all occurances of @pkgdatadir@ with the $pkgdatadir
 	  variable that is set by configure. This ensures that the file
 	  paths in the manpage arp-scan.1 reflect the paths on the system
 	  where ./configure was run.
-	* arp-scan.1: removed
-	* arp-scan.1.dist: New file
+	* arp-scan.1: renamed to arp-scan.1.dist with PKGDATADIR substitution.
 
-2021-10-27 Roy Hills <royhills@hotmail.com>
+2022-10-27 Roy Hills <royhills@hotmail.com>
 
 	* arp-fingerprint.1, arp-scan.1, get-iab.1, get-oui.1, mac-vendor.5:
 	  Standardised man page fonts, corrected some inaccuracies and removed
 	  author section.
 
 	* Updated ieee-oui.txt and ieee-iab.txt files.
 
-2021-10-25 Roy Hills <royhills@hotmail.com>
+2022-10-25 Roy Hills <royhills@hotmail.com>
 
 	* format.c: New file containing output format functions.
 
@@ -27,7 +59,7 @@
 	* arp-scan.c, arp-scan.h, arp-scan.1: Add --format option and output
 	  fields processing code.
 
-2021-10-24 Roy Hills <royhills@hotmail.com>
+2022-10-24 Roy Hills <royhills@hotmail.com>
 
 	* Makefile.am, check-decode: Added checks to test output format with
 	  --rtt, --resolve, different ARP/header address, and locally
@@ -36,7 +68,7 @@
 	* pkt-diff-frame-addr.pcap, pkt-local-admin.pcap: New files for
 	  above tests.
 
-2021-10-23 Roy Hills <royhills@hotmail.com>
+2022-10-23 Roy Hills <royhills@hotmail.com>
 
 	* arp-scan.c, error.c arp-scan.c, error.c: Changed most fixed-length
 	  strings to use dynamic length strings.
@@ -45,7 +77,7 @@
 	  according to POSIX. No need to accomodate bugs in ancient glibc
 	  versions.
 
-2021-10-09 Roy Hills <royhills@hotmail.com>
+2022-10-09 Roy Hills <royhills@hotmail.com>
 
 	* arp-scan.c: Added POSIX.1e capabilities support.
 

Makefile.am

@@ -6,20 +6,22 @@ bin_PROGRAMS = arp-scan
 #
 dist_bin_SCRIPTS = get-oui get-iab arp-fingerprint
 #
-dist_check_SCRIPTS = check-run1 check-packet check-decode check-host-list
+dist_check_SCRIPTS = check-run1 check-packet check-decode check-host-list check-ieee-reg
 #
-dist_man_MANS = arp-scan.1 get-oui.1 get-iab.1 arp-fingerprint.1 mac-vendor.5
+dist_man_MANS = arp-scan.1 get-oui.1 arp-fingerprint.1 mac-vendor.5
 #
 arp_scan_SOURCES = arp-scan.c arp-scan.h error.c wrappers.c utils.c mt19937ar.c format.c mt19937ar.h
 arp_scan_LDADD = $(LIBOBJS)
 #
-dist_pkgdata_DATA = ieee-oui.txt ieee-iab.txt mac-vendor.txt
+dist_pkgdata_DATA = ieee-oui.txt mac-vendor.txt
 #
 TESTS = $(dist_check_SCRIPTS)
-EXTRA_DIST = arp-scan.1.dist pkt-simple-request.dat pkt-custom-request.dat pkt-custom-request-padding.dat pkt-custom-request-llc.dat pkt-custom-request-vlan.dat pkt-simple-response.pcap pkt-padding-response.pcap pkt-vlan-response.pcap pkt-llc-response.pcap pkt-net1921681-response.pcap pkt-trailer-response.pcap pkt-vlan-llc-response.pcap pkt-custom-request-vlan-llc.dat pkt-dup-response.pcap pkt-diff-frame-addr.pcap pkt-local-admin.pcap
+EXTRA_DIST = arp-scan.1.dist get-oui.1.dist pkt-simple-request.dat pkt-custom-request.dat pkt-custom-request-padding.dat pkt-custom-request-llc.dat pkt-custom-request-vlan.dat pkt-simple-response.pcap pkt-padding-response.pcap pkt-vlan-response.pcap pkt-llc-response.pcap pkt-net1921681-response.pcap pkt-trailer-response.pcap pkt-vlan-llc-response.pcap pkt-custom-request-vlan-llc.dat pkt-dup-response.pcap pkt-diff-frame-addr.pcap pkt-local-admin.pcap pkt-ieee-regcheck.pcap
 #
 # Substitute autoconf pkgdatadir variable in arp-scan.1 manpage
-CLEANFILES = arp-scan.1
+CLEANFILES = arp-scan.1 get-oui.1
 do_subst = sed -e 's,[@]PKGDATADIR[@],$(pkgdatadir),g'
 arp-scan.1: arp-scan.1.dist Makefile
 	$(do_subst) < $(srcdir)/arp-scan.1.dist > arp-scan.1
+get-oui.1: get-oui.1.dist Makefile
+	$(do_subst) < $(srcdir)/get-oui.1.dist > get-oui.1

NEWS

@@ -6,7 +6,11 @@ YYYY-MM-DD arp-scan 1.9.9 (in progress)
 * New Features:
 
   - Added POSIX.1e capabilities support for systems with libcap (Linux only)
-  - --format option provides flexible output format (see manpage or help)
+  - New --format option provides flexible output format (see manpage or help)
+  - get-oui script now stores IEEE registries MA-L (OUI), MA-M, MA-S (OUI36)
+    and IAB in the single file ieee-oui.txt. The ieee-iab.txt file and the
+    --iabfiles option to arp-scan have been removed as they are no longer
+    required.
 
 2021-10-08 arp-scan 1.9.8 (git tag 1.9.8)
 

README.md

@@ -25,7 +25,7 @@ arp-scan uses the GNU automake and autoconf tools, so the typical installation p
 
 You will need GNU automake and autoconf, the make utility, an ANSI C compiler (tested with gcc and clang), the development header files and libraries, and libpcap version 1.5 or later. On Linux, it is recommended to install the libcap POSIX.1e capability support development header files and libraries (typically in a package called `libcap-dev`, `libcap-devel` or similar) so arp-scan can be made capabilities-aware.
 
-If you want to run the Perl scripts arp-fingerprint, get-oui and get-iab, you will need to have the Perl interpreter installed.  In addition, for get-oui and get-iab, you will need the LWP::UserAgent Perl module.
+If you want to run the Perl scripts arp-fingerprint and get-oui, you will need to have the Perl interpreter installed.  In addition, for get-oui, you will need the LWP::UserAgent and Text::CSV Perl modules.
 
 You can pass various options to "configure" to control the build and installation process. Run "./configure --help" to see a list of options. arp-scan has one package-specific configure option:
 
@@ -56,4 +56,4 @@ For usage information, including details of all the options, use:
 
 ```arp-scan --help```
 
-For more detailed documentation, see the manual pages: arp-scan(1), arp-fingerprint(1), get-iab(1), get-oui(1) and mac-vendor(5).
+For more detailed documentation, see the manual pages: arp-scan(1), arp-fingerprint(1), get-oui(1) and mac-vendor(5).

arp-scan.1.dist

@@ -266,32 +266,26 @@ is not always the same order as the requests were sent because some
 hosts may respond faster than others.
 .PP
 The vendor decoding uses the files
-.IR ieee-oui.txt ,
-.I ieee-iab.txt
+.I ieee-oui.txt
 and
 .IR mac-vendor.txt ,
 which are supplied with
 .BR arp-scan .
 The
 .I ieee-oui.txt
-and
-.I ieee-iab.txt
-files are generated from the OUI and IAB data on the IEEE website at
-.I http://standards-oui.ieee.org/oui/oui.txt
-and
-.IR http://standards-oui.ieee.org/iab/iab.txt .
-The Perl scripts
+file is generated from the \fIMA-L\fP (OUI), \fIMA-M\fP, \fIMA-S\fP (OUI36) and \fIIAB\fP registries. See
+.I https://standards.ieee.org/products-programs/regauth/
+for details of these registries.
+The Perl script
 .B get-oui
-and
-.BR get-iab ,
-which are included in the
+which is included in the
 .B arp-scan
-package, can be used to update these
-files with the latest data from the IEEE website.
+package can be used to update this file with the latest data from the IEEE
+website.
 The
 .I mac-vendor.txt
 file contains other MAC to Vendor mappings that are not covered by
-the IEEE OUI and IAB files, and can be used to add custom mappings.
+the IEEE OUI file, and can be used to add custom mappings.
 .PP
 Almost all hosts that support IP will respond to
 .B arp-scan
@@ -482,13 +476,6 @@ is \fIieee-oui.txt\fP in the current directory. If that is
 not found, then the file
 \fI@PKGDATADIR@/ieee-oui.txt\fP is used.
 .TP
-\fB--iabfile\fP=\fI<s>\fP or \fB-O \fI<s>\fR
-Use IEEE Ethernet IAB to vendor mapping file \fI<s>\fP.
-If this option is not specified, the default filename
-is \fIieee-iab.txt\fP in the current directory. If that is
-not found, then the file
-\fI@PKGDATADIR@/ieee-iab.txt\fP is used.
-.TP
 \fB--macfile\fP=\fI<s>\fP or \fB-O \fI<s>\fR
 Use custom Ethernet MAC to vendor mapping file \fI<s>\fP.
 If this option is not specified, the default filename
@@ -687,9 +674,6 @@ Example: --format='${ip}\\t${mac}\\t${vendor}'
 .I @PKGDATADIR@/ieee-oui.txt
 List of IEEE OUI (Organisationally Unique Identifier) to vendor mappings.
 .TP
-.I @PKGDATADIR@/ieee-iab.txt
-List of IEEE IAB (Individual Address Block) to vendor mappings.
-.TP
 .I @PKGDATADIR@/mac-vendor.txt
 List of other Ethernet MAC to vendor mappings.
 .SH EXAMPLES
@@ -752,8 +736,6 @@ Ending arp-scan 1.4: 8 hosts scanned in 0.820 seconds (9.76 hosts/sec).  4 respo
 .SH "SEE ALSO"
 .BR get-oui (1)
 .PP
-.BR get-iab (1)
-.PP
 .BR arp-fingerprint (1)
 .PP
 .B RFC 826

arp-scan.c

@@ -62,7 +62,6 @@ static int arp_spa_is_tpa=0;		/* Source IP is dest IP */
 static unsigned char arp_sha[ETH_ALEN];	/* Source Ethernet MAC Address */
 static int arp_sha_flag=0;		/* Source MAC address specified */
 static char *ouifilename = NULL;	/* OUI filename */
-static char *iabfilename = NULL;	/* IAB filename */
 static char *macfilename = NULL;	/* MAC filename */
 static char *pcap_savefile = NULL;	/* pcap savefile filename */
 static int arp_op=DEFAULT_ARP_OP;	/* ARP Operation code */
@@ -377,13 +376,6 @@ main(int argc, char *argv[]) {
                   count, fn);
       free(fn);
 
-      fn = get_mac_vendor_filename(iabfilename, PKGDATADIR, IABFILENAME);
-      count = add_mac_vendor(fn);
-      if (verbose > 1 && count > 0)
-         warn_msg("DEBUG: Loaded %d IEEE IAB/Vendor entries from %s.",
-                  count, fn);
-      free(fn);
-
       fn = get_mac_vendor_filename(macfilename, PKGDATADIR, MACFILENAME);
       count = add_mac_vendor(fn);
       if (verbose > 1 && count > 0)
@@ -1318,11 +1310,6 @@ usage(int status, int detailed) {
       fprintf(stdout, "\t\t\tis %s in the current directory. If that is\n", OUIFILENAME);
       fprintf(stdout, "\t\t\tnot found, then the file\n");
       fprintf(stdout, "\t\t\t%s/%s is used.\n", PKGDATADIR, OUIFILENAME);
-      fprintf(stdout, "\n--iabfile=<s> or -O <s>\tUse IEEE Ethernet IAB to vendor mapping file <s>.\n");
-      fprintf(stdout, "\t\t\tIf this option is not specified, the default filename\n");
-      fprintf(stdout, "\t\t\tis %s in the current directory. If that is\n", IABFILENAME);
-      fprintf(stdout, "\t\t\tnot found, then the file\n");
-      fprintf(stdout, "\t\t\t%s/%s is used.\n", PKGDATADIR, IABFILENAME);
       fprintf(stdout, "\n--macfile=<s> or -O <s>\tUse custom Ethernet MAC to vendor mapping file <s>.\n");
       fprintf(stdout, "\t\t\tIf this option is not specified, the default filename\n");
       fprintf(stdout, "\t\t\tis %s in the current directory. If that is\n", MACFILENAME);
@@ -2062,7 +2049,6 @@ process_options(int argc, char *argv[]) {
       {"numeric", no_argument, 0, 'N'},
       {"bandwidth", required_argument, 0, 'B'},
       {"ouifile", required_argument, 0, 'O'},
-      {"iabfile", required_argument, 0, 'F'},
       {"macfile", required_argument, 0, 'm'},
       {"arpspa", required_argument, 0, 's'},
       {"arpop", required_argument, 0, 'o'},
@@ -2094,7 +2080,7 @@ process_options(int argc, char *argv[]) {
  * available short option characters:
  *
  * lower:       --c-e----j---------------z
- * UPPER:       --C---G--JK---------U--X-Z
+ * UPPER:       --C--FG--JK---------U--X-Z
  * Digits:      0123456789
  */
    const char *short_options =
@@ -2163,9 +2149,6 @@ process_options(int argc, char *argv[]) {
          case 'O':	/* --ouifile */
             ouifilename = make_message("%s", optarg);
             break;
-         case 'F':	/* --iabfile */
-            iabfilename = make_message("%s", optarg);
-            break;
          case 'm':	/* --macfile */
             macfilename = make_message("%s", optarg);
             break;

arp-scan.h

@@ -156,7 +156,6 @@
 #define ETH_P_IP 0x0800			/* Internet Protocol packet */
 #define ETH_P_ARP 0x0806		/* Address Resolution packet */
 #define OUIFILENAME "ieee-oui.txt"	/* Default IEEE OUI filename */
-#define IABFILENAME "ieee-iab.txt"	/* Default IEEE IAB filename */
 #define MACFILENAME "mac-vendor.txt"	/* Default MAC/Vendor filename */
 #define DEFAULT_ARP_OP ARPOP_REQUEST	/* Default ARP operation */
 #define DEFAULT_ARP_HRD ARPHRD_ETHER	/* Default ARP hardware type */
@@ -169,7 +168,7 @@
 #define OPT_WRITEPKTTOFILE 256		/* --writepkttofile option */
 #define OPT_READPKTFROMFILE 257		/* --readpktfromfile option */
 #define OPT_RANDOMSEED 258		/* --randomseed option */
-#define HASH_TABLE_SIZE 50000		/* Max size of OUI/Vendor hash table */
+#define HASH_TABLE_SIZE 70000		/* Max size of MAC/Vendor hash table */
 #define DEFAULT_RETRY_SEND 20		/* Default no. of send packet retries */
 #define DEFAULT_RETRY_SEND_INTERVAL 5000  /* Default interval between send
                                         * packet retries in microseconds */