From cb1991739e4185321c4196cf123e05d0e5cb3428 Mon Sep 17 00:00:00 2001
From: Athanasius <github@miggy.org>
Date: Thu, 10 Feb 2022 15:46:47 +0000
Subject: [PATCH 1/4] scripts/apache-log-rate: Determine highest message rate
 from apache logs

---
 scripts/apache-log-rate | 98 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)
 create mode 100755 scripts/apache-log-rate

diff --git a/scripts/apache-log-rate b/scripts/apache-log-rate
new file mode 100755
index 0000000..d5e9106
--- /dev/null
+++ b/scripts/apache-log-rate
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+# vim: wrapmargin=0 textwidth=0 smarttab expandtab tabstop=2 shiftwidth=2
+"""Process Apache access.log lines to find highest rate of /upload/."""
+
+import argparse
+import datetime
+import dateutil.parser
+import fileinput
+import re
+
+
+def process_log_file(
+  input_file: str ='-',
+  request_text: str = '/upload/',
+  window_size: int = 1,
+) -> None:
+  """
+  Process the indicated log file to determine peak rate of interesting lines.
+
+  :param input_file: Name of input file, `-` for stdin
+  :param request_text: The text that denotes an interesting line
+  :param window_size: Time, in seconds, for the window to assess
+  """
+  print(f'With:\n\tinput_file: "{input_file}"\n\trequest_text: "{request_text}"')
+  with fileinput.FileInput(files=(input_file)) as f:
+    apache_re = re.compile(r'^(?P<host>[.:0-9a-fA-F]{3,39}) - - \[(?P<datetime>[^\]]+)\] (?P<logtext>.*' + request_text + '.*)$')
+    apache_datetime_re = re.compile(
+      r'^(?P<d>[0-9]{2})/(?P<mon>[^/]{3})/(?P<YYYY>[0-9]{4}):(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4})$'
+    )
+
+    window_time_delta = datetime.timedelta(seconds=window_size)
+    window_count = 0
+    window_dts = []
+    line_count = 0
+    for line in f:
+      matches = apache_re.search(line)
+      if matches:
+        line_count += 1
+        # print(f'\nMatches:\n{line}')
+        # This will be referenced so many times we want a short name
+        m = apache_datetime_re.search(matches.group('datetime'))
+        this_dt_iso8601 = f'{m.group("YYYY")}-{m.group("mon")}-{m.group("d")} {m.group("time")}'
+
+        this_dt = dateutil.parser.parse(this_dt_iso8601)
+        # print(f'Timestamp: {this_dt}')
+        window_dts.append(this_dt)
+
+        # Now we need to expire any of the oldest stored timestamps that
+        # are outside the window relative to this
+        window_dts = list(filter(lambda dt: dt >= this_dt - window_time_delta, window_dts))
+        if len(window_dts) > window_count:
+          window_count = len(window_dts)
+
+        # print()
+
+      else:
+        # print(f'\nNo matches:\n{line}\n')
+        pass
+
+    print(f'With window size         : {window_size:>9}')
+    print(f'Total line matching lines: {line_count:>9}')
+    print(f'Largest window count     : {window_count:>9} ({window_count / window_size:>9}/s)')
+    
+
+def main():
+  parser = argparse.ArgumentParser(
+    description='Process Apache web server access.log lines, counting the number of a specific request per a unit of time.',
+  )
+  
+  parser.add_argument(
+    '--window-length',
+    metavar='<window size in seconds>',
+    required=False,
+    default=1,
+    help='The time period in which the max rate will be.',
+  )
+
+  parser.add_argument(
+    'input_file',
+    metavar='<input file name>',
+    help='Name of an Apache access.log file.  You may use "-" for standard input.',
+  )
+
+  parser.add_argument(
+    'request_text',
+    metavar='<per-request text selector>',
+    help='Text that appears in the log lines of interest.  Defaults to "/upload/"',
+    nargs='?',
+    default='/upload/',
+  )
+
+  args = parser.parse_args()
+
+  process_log_file(input_file=args.input_file, request_text=args.request_text, window_size=int(args.window_length))
+
+
+if __name__ == '__main__':
+  main()

From 42129a50cebae2037375d4a35854cc1db04a8033 Mon Sep 17 00:00:00 2001
From: Athanasius <github@miggy.org>
Date: Thu, 10 Feb 2022 16:17:49 +0000
Subject: [PATCH 2/4] scripts/apache-log-rate: Attempt to speed up with an
 array slice

---
 scripts/apache-log-rate | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/scripts/apache-log-rate b/scripts/apache-log-rate
index d5e9106..f41915b 100755
--- a/scripts/apache-log-rate
+++ b/scripts/apache-log-rate
@@ -45,9 +45,20 @@ def process_log_file(
         # print(f'Timestamp: {this_dt}')
         window_dts.append(this_dt)
 
+        # Find the oldest entry that is still within the window:
+        oldest_of_interest = this_dt - window_time_delta
+        dti = 0
+        for dt in window_dts:
+          if dt >= oldest_of_interest:
+            break
+          dti += 1
+
+        window_dts = window_dts[dti:]
+
         # Now we need to expire any of the oldest stored timestamps that
         # are outside the window relative to this
-        window_dts = list(filter(lambda dt: dt >= this_dt - window_time_delta, window_dts))
+        # window_dts = list(filter(lambda dt: dt >= this_dt - window_time_delta, window_dts))
+
         if len(window_dts) > window_count:
           window_count = len(window_dts)
 

From 0414b18f97cff42a11f2bd9b81a0e4f7df24ee19 Mon Sep 17 00:00:00 2001
From: Athanasius <github@miggy.org>
Date: Thu, 10 Feb 2022 16:26:25 +0000
Subject: [PATCH 3/4] scripts/apache-log-rate: Use a `deque` instead of a list

---
 scripts/apache-log-rate | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/scripts/apache-log-rate b/scripts/apache-log-rate
index f41915b..65ac301 100755
--- a/scripts/apache-log-rate
+++ b/scripts/apache-log-rate
@@ -3,6 +3,7 @@
 """Process Apache access.log lines to find highest rate of /upload/."""
 
 import argparse
+from collections import deque
 import datetime
 import dateutil.parser
 import fileinput
@@ -30,7 +31,7 @@ def process_log_file(
 
     window_time_delta = datetime.timedelta(seconds=window_size)
     window_count = 0
-    window_dts = []
+    window_dts = deque()
     line_count = 0
     for line in f:
       matches = apache_re.search(line)
@@ -47,13 +48,8 @@ def process_log_file(
 
         # Find the oldest entry that is still within the window:
         oldest_of_interest = this_dt - window_time_delta
-        dti = 0
-        for dt in window_dts:
-          if dt >= oldest_of_interest:
-            break
-          dti += 1
-
-        window_dts = window_dts[dti:]
+        while window_dts[0] < oldest_of_interest:
+          window_dts.popleft()
 
         # Now we need to expire any of the oldest stored timestamps that
         # are outside the window relative to this
@@ -61,6 +57,7 @@ def process_log_file(
 
         if len(window_dts) > window_count:
           window_count = len(window_dts)
+          # print(f'Largest window count     : {window_count:>9} ({window_count / window_size:>9}/s)')
 
         # print()
 

From 2b653733d2a6f909ec7cb92fe88788197db1a068 Mon Sep 17 00:00:00 2001
From: Athanasius <github@miggy.org>
Date: Thu, 10 Feb 2022 19:25:53 +0000
Subject: [PATCH 4/4] scripts/apache-log-rate: Bug fix & misc.

* Comparison had an off-by one, accidentally including "second before"
  still.
* Print out compiled regex for matching apache lines, in case of custom
  request text usage.
* Output the timestamp at the end of the largest window.  This enables
  easy checking of logs for possible anomalies around then.
* Comment about how apache logs *aren't* actually strictly in
  time-order, but we assume they are.
---
 scripts/apache-log-rate | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/scripts/apache-log-rate b/scripts/apache-log-rate
index 65ac301..5e63280 100755
--- a/scripts/apache-log-rate
+++ b/scripts/apache-log-rate
@@ -25,12 +25,15 @@ def process_log_file(
   print(f'With:\n\tinput_file: "{input_file}"\n\trequest_text: "{request_text}"')
   with fileinput.FileInput(files=(input_file)) as f:
     apache_re = re.compile(r'^(?P<host>[.:0-9a-fA-F]{3,39}) - - \[(?P<datetime>[^\]]+)\] (?P<logtext>.*' + request_text + '.*)$')
+    print(f'Apache RE:\n{apache_re}\n')
     apache_datetime_re = re.compile(
       r'^(?P<d>[0-9]{2})/(?P<mon>[^/]{3})/(?P<YYYY>[0-9]{4}):(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4})$'
     )
 
     window_time_delta = datetime.timedelta(seconds=window_size)
     window_count = 0
+    last_dt = None
+    window_end_longest_count = None
     window_dts = deque()
     line_count = 0
     for line in f:
@@ -42,23 +45,32 @@ def process_log_file(
         m = apache_datetime_re.search(matches.group('datetime'))
         this_dt_iso8601 = f'{m.group("YYYY")}-{m.group("mon")}-{m.group("d")} {m.group("time")}'
 
+        ###############################################################
+        # This code absolutely assumes that the apache log lines are
+        # in strictly increasing time sequence order.
+        #
+        # That's not necessarily true.  It has been observed that e.g.
+        # a long line for 00:24:39 can occur in the middle of lines for
+        # 00:24:40.
+        #
+        # Hopefully this doesn't happen too much.
+        ###############################################################
         this_dt = dateutil.parser.parse(this_dt_iso8601)
         # print(f'Timestamp: {this_dt}')
         window_dts.append(this_dt)
 
         # Find the oldest entry that is still within the window:
         oldest_of_interest = this_dt - window_time_delta
-        while window_dts[0] < oldest_of_interest:
+        while window_dts[0] <= oldest_of_interest:
           window_dts.popleft()
 
-        # Now we need to expire any of the oldest stored timestamps that
-        # are outside the window relative to this
-        # window_dts = list(filter(lambda dt: dt >= this_dt - window_time_delta, window_dts))
-
         if len(window_dts) > window_count:
           window_count = len(window_dts)
+          window_end_longest_count = last_dt
           # print(f'Largest window count     : {window_count:>9} ({window_count / window_size:>9}/s)')
 
+        last_dt = this_dt
+
         # print()
 
       else:
@@ -68,6 +80,7 @@ def process_log_file(
     print(f'With window size         : {window_size:>9}')
     print(f'Total line matching lines: {line_count:>9}')
     print(f'Largest window count     : {window_count:>9} ({window_count / window_size:>9}/s)')
+    print(f'Busiest window ended at: {window_end_longest_count.strftime("%d/%b/%Y:%H:%M:%S")}')
     
 
 def main():