TLS: improve reassembler (#1669)

* TLS: cosmetic changes

* TLS: improve reassembler

We might need to contemporary re-order messages from both directions:
use one buffer per direction.

9 files changed, 45 insertions, 4 deletions

diff --git a/tests/pcap/tls_2_reasms.pcapng b/tests/pcap/tls_2_reasms.pcapng
new file mode 100644
index 000000000..3275191b2
--- /dev/null
+++ b/tests/pcap/tls_2_reasms.pcapng
diff --git a/tests/pcap/tls_2_reasms_b.pcapng b/tests/pcap/tls_2_reasms_b.pcapng
new file mode 100644
index 000000000..f50024974
--- /dev/null
+++ b/tests/pcap/tls_2_reasms_b.pcapng
diff --git a/tests/pcap/tls_missing_ch_frag.pcap b/tests/pcap/tls_missing_ch_frag.pcap
new file mode 100644
index 000000000..edd0529ca
--- /dev/null
+++ b/tests/pcap/tls_missing_ch_frag.pcap
diff --git a/tests/result/skype.pcap.out b/tests/result/skype.pcap.out
index 8ddb2020a..eae5ec38f 100644
--- a/tests/result/skype.pcap.out
+++ b/tests/result/skype.pcap.out
@@ -7,7 +7,7 @@ Confidence Unknown          : 61 (flows)
 Confidence Match by port    : 27 (flows)
 Confidence Match by IP      : 1 (flows)
 Confidence DPI              : 204 (flows)
-Num dissector calls: 31670 (108.09 diss/flow)
+Num dissector calls: 31667 (108.08 diss/flow)
 
 Unknown	1575	272476	61
 DNS	2	267	1
diff --git a/tests/result/skype_no_unknown.pcap.out b/tests/result/skype_no_unknown.pcap.out
index 32ad91911..7f8e32dce 100644
--- a/tests/result/skype_no_unknown.pcap.out
+++ b/tests/result/skype_no_unknown.pcap.out
@@ -1,6 +1,6 @@
 Guessed flow protos:	72
 
-DPI Packets (TCP):	1111	(14.62 pkts/flow)
+DPI Packets (TCP):	1120	(14.74 pkts/flow)
 DPI Packets (UDP):	288	(1.55 pkts/flow)
 DPI Packets (other):	5	(1.00 pkts/flow)
 Confidence Unknown          : 45 (flows)
diff --git a/tests/result/teams.pcap.out b/tests/result/teams.pcap.out
index 8e52e12bf..7f733e28e 100644
--- a/tests/result/teams.pcap.out
+++ b/tests/result/teams.pcap.out
@@ -1,6 +1,6 @@
 Guessed flow protos:	4
 
-DPI Packets (TCP):	356	(8.48 pkts/flow)
+DPI Packets (TCP):	347	(8.26 pkts/flow)
 DPI Packets (UDP):	87	(2.17 pkts/flow)
 DPI Packets (other):	1	(1.00 pkts/flow)
 Confidence Unknown          : 1 (flows)
@@ -35,7 +35,7 @@ JA3 Host Stats:
 	3	TCP 192.168.1.6:60532 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][Encrypted][Confidence: DPI][cat: Cloud/13][49 pkts/58592 bytes <-> 28 pkts/6555 bytes][Goodput ratio: 94/72][0.71 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: 0.799 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 13/29 177/221 32/57][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1196/234 1494/1506 564/435][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: a1674500365bdd882188db63730e69a2][ServerNames: *.events.data.microsoft.com,events.data.microsoft.com,*.pipe.aria.microsoft.com,pipe.skype.com,*.pipe.skype.com,*.mobile.events.data.microsoft.com,mobile.events.data.microsoft.com,*.events.data.msn.com,events.data.msn.com][JA3S: ae4edc6faf64d08308082ad26be60767][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 4][Subject: CN=*.events.data.microsoft.com][Certificate SHA-1: 33:B3:B7:E9:DA:25:F5:A0:04:E9:63:87:B6:FB:54:77:DB:ED:27:EB][Safari][Validity: 2019-10-10 21:55:38 - 2021-10-10 21:55:38][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 2,2,2,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,81,4,0,0]
 	4	TCP 192.168.1.6:60554 <-> 52.113.194.132:443 [proto: 91.250/TLS.Teams][Encrypted][Confidence: DPI][cat: Collaborative/15][24 pkts/2746 bytes <-> 28 pkts/30546 bytes][Goodput ratio: 52/95][0.23 sec][Hostname/SNI: config.teams.microsoft.com][bytes ratio: -0.835 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4/9 21/140 7/29][Pkt Len c2s/s2c min/avg/max/stddev: 54/60 114/1091 1136/1506 217/607][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: e4d448cdfe06dc1243c1eb026c74ac9a][ServerNames: *.config.teams.microsoft.com,config.teams.microsoft.com][JA3S: 7d8fd34fdb13a7fff30d5a52846b6c4c][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 1][Subject: CN=config.teams.microsoft.com][Certificate SHA-1: B9:54:54:12:C9:E9:43:65:10:70:04:7B:AD:B6:0C:46:06:38:A5:FA][Firefox][Validity: 2019-12-11 02:04:20 - 2021-12-11 02:04:20][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 0,7,0,3,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,3,0,0,3,0,0,0,0,0,7,0,0,0,0,0,65,0,0]
 	5	TCP 192.168.1.6:60561 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][Encrypted][Confidence: DPI][cat: Cloud/13][23 pkts/19184 bytes <-> 14 pkts/5643 bytes][Goodput ratio: 92/83][0.82 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: 0.545 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 25/44 161/136 43/48][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 834/403 1494/1506 690/567][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: a1674500365bdd882188db63730e69a2][ServerNames: *.events.data.microsoft.com,events.data.microsoft.com,*.pipe.aria.microsoft.com,pipe.skype.com,*.pipe.skype.com,*.mobile.events.data.microsoft.com,mobile.events.data.microsoft.com,*.events.data.msn.com,events.data.msn.com][JA3S: ae4edc6faf64d08308082ad26be60767][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 4][Subject: CN=*.events.data.microsoft.com][Certificate SHA-1: 33:B3:B7:E9:DA:25:F5:A0:04:E9:63:87:B6:FB:54:77:DB:ED:27:EB][Safari][Validity: 2019-10-10 21:55:38 - 2021-10-10 21:55:38][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 4,4,4,0,0,0,9,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,60,9,0,0]
-	6	TCP 192.168.1.6:60535 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][Encrypted][Confidence: DPI][cat: Cloud/13][21 pkts/16793 bytes <-> 13 pkts/5565 bytes][Goodput ratio: 92/84][0.33 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: 0.502 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/18 48/49 17/20][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 800/428 1494/1506 681/581][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: a1674500365bdd882188db63730e69a2][ServerNames: *.events.data.microsoft.com,events.data.microsoft.com,*.pipe.aria.microsoft.com,pipe.skype.com,*.pipe.skype.com,*.mobile.events.data.microsoft.com,mobile.events.data.microsoft.com,*.events.data.msn.com,events.data.msn.com][JA3S: ae4edc6faf64d08308082ad26be60767][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 4][Subject: CN=*.events.data.microsoft.com][Certificate SHA-1: 33:B3:B7:E9:DA:25:F5:A0:04:E9:63:87:B6:FB:54:77:DB:ED:27:EB][Safari][Validity: 2019-10-10 21:55:38 - 2021-10-10 21:55:38][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 5,5,5,0,0,0,5,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,53,10,0,0]
+	6	TCP 192.168.1.6:60535 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][Encrypted][Confidence: DPI][cat: Cloud/13][21 pkts/16793 bytes <-> 13 pkts/5565 bytes][Goodput ratio: 92/84][0.33 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: 0.502 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/18 48/49 17/20][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 800/428 1494/1506 681/581][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: a1674500365bdd882188db63730e69a2][Safari][Plen Bins: 5,5,5,0,0,0,5,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,53,10,0,0]
 	7	TCP 192.168.1.6:60559 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][Encrypted][Confidence: DPI][cat: Cloud/13][21 pkts/15525 bytes <-> 12 pkts/5499 bytes][Goodput ratio: 91/85][0.35 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: 0.477 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/21 52/51 22/22][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 739/458 1494/1506 682/595][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: a1674500365bdd882188db63730e69a2][Safari][Plen Bins: 5,5,5,0,0,0,5,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,5,0,0,52,11,0,0]
 	8	TCP 192.168.1.6:60545 <-> 52.114.77.58:443 [proto: 91.250/TLS.Teams][Encrypted][Confidence: DPI][cat: Collaborative/15][49 pkts/7568 bytes <-> 34 pkts/11426 bytes][Goodput ratio: 65/83][9.23 sec][Hostname/SNI: presence.teams.microsoft.com][ALPN: h2;http/1.1][bytes ratio: -0.203 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 226/294 4927/4971 803/983][Pkt Len c2s/s2c min/avg/max/stddev: 54/60 154/336 1494/1506 217/458][TLSv1.2][JA3C: ebf5e0e525258d7a8dcb54aa1564ecbd][Plen Bins: 0,21,17,10,8,6,4,0,6,2,0,0,2,6,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,0,0]
 	9	TCP 192.168.1.6:60549 <-> 13.107.18.11:443 [proto: 91.219/TLS.Microsoft365][Encrypted][Confidence: DPI][cat: Collaborative/15][28 pkts/7696 bytes <-> 26 pkts/9797 bytes][Goodput ratio: 80/85][1.16 sec][Hostname/SNI: substrate.office.com][ALPN: h2;http/1.1][bytes ratio: -0.120 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 47/23 539/167 115/43][Pkt Len c2s/s2c min/avg/max/stddev: 54/60 275/377 1494/1506 397/471][TLSv1.2][JA3C: ebf5e0e525258d7a8dcb54aa1564ecbd][ServerNames: outlook.office.com,attachment.outlook.office.net,attachment.outlook.officeppe.net,bookings.office.com,delve.office.com,edge.outlook.office365.com,edgesdf.outlook.com,img.delve.office.com,outlook.live.com,outlook-sdf.live.com,outlook-sdf.office.com,sdfedge-pilot.outlook.com,substrate.office.com,substrate-sdf.office.com,afd-k-acdc-direct.office.com,beta-sdf.yammer.com,teams-sdf.yammer.com,beta.yammer.com,teams.yammer.com,attachments.office.net,attachments-sdf.office.net,afd-k.office.com,afd-k-sdf.office.com][JA3S: a66ea560599a2f5c89eec8c3a0d69cee][Issuer: C=US, O=DigiCert Inc, CN=DigiCert Cloud Services CA-1][Subject: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, CN=Outlook.office.com][Certificate SHA-1: AA:D3:F5:66:06:48:AA:F8:8E:9B:79:D6:7F:1D:53:EA:3F:97:03:A2][Validity: 2019-07-12 00:00:00 - 2021-07-12 12:00:00][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 0,22,7,0,7,0,7,0,0,3,3,0,0,0,3,0,7,0,3,0,10,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18,0,0]
diff --git a/tests/result/tls_2_reasms.pcapng.out b/tests/result/tls_2_reasms.pcapng.out
new file mode 100644
index 000000000..ef5621be8
--- /dev/null
+++ b/tests/result/tls_2_reasms.pcapng.out
@@ -0,0 +1,14 @@
+Guessed flow protos:	0
+
+DPI Packets (TCP):	4	(4.00 pkts/flow)
+Confidence DPI              : 1 (flows)
+Num dissector calls: 7 (7.00 diss/flow)
+
+Instagram	14	6907	1
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+	1	 192.91.186.174           	 1      
+
+
+	1	TCP 192.91.186.174:443 <-> 25.137.80.32:38134 [proto: 91.211/TLS.Instagram][Encrypted][Confidence: DPI][cat: SocialNetwork/6][11 pkts/4419 bytes <-> 3 pkts/2488 bytes][Goodput ratio: 83/92][0.95 sec][Hostname/SNI: i.instagram.com][ALPN: h2;h2-fb;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.3 (Fizz)][bytes ratio: 0.280 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/15 105/232 465/449 152/217][Pkt Len c2s/s2c min/avg/max/stddev: 74/470 402/829 1414/1414 483/417][TLSv1.3 (Fizz)][JA3C: 44dab16d680ef93487bc16ad23b3ffb1][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 7,0,24,7,0,7,7,7,0,0,0,0,7,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24,0,0,0,0,0]
diff --git a/tests/result/tls_2_reasms_b.pcapng.out b/tests/result/tls_2_reasms_b.pcapng.out
new file mode 100644
index 000000000..5e96556c8
--- /dev/null
+++ b/tests/result/tls_2_reasms_b.pcapng.out
@@ -0,0 +1,14 @@
+Guessed flow protos:	0
+
+DPI Packets (TCP):	5	(5.00 pkts/flow)
+Confidence DPI              : 1 (flows)
+Num dissector calls: 11 (11.00 diss/flow)
+
+Facebook	15	13455	1
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+	1	 88.14.137.195            	 1      
+
+
+	1	TCP 88.14.137.195:443 <-> 196.234.165.216:37658 [proto: 91.119/TLS.Facebook][Encrypted][Confidence: DPI][cat: SocialNetwork/6][12 pkts/11078 bytes <-> 3 pkts/2377 bytes][Goodput ratio: 93/92][1.05 sec][Hostname/SNI: video.fmct2-3.fna.fbcdn.net][ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.3 (Fizz)][bytes ratio: 0.647 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/4 105/8 1002/12 299/4][Pkt Len c2s/s2c min/avg/max/stddev: 74/478 923/792 1414/1414 599/440][TLSv1.3 (Fizz)][JA3C: 44dab16d680ef93487bc16ad23b3ffb1][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 7,0,0,0,0,0,7,0,0,0,0,0,7,7,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,0,0,0,0,0]
diff --git a/tests/result/tls_missing_ch_frag.pcap.out b/tests/result/tls_missing_ch_frag.pcap.out
new file mode 100644
index 000000000..f6d4181e8
--- /dev/null
+++ b/tests/result/tls_missing_ch_frag.pcap.out
@@ -0,0 +1,13 @@
+Guessed flow protos:	0
+
+DPI Packets (TCP):	3	(3.00 pkts/flow)
+Confidence DPI              : 1 (flows)
+Num dissector calls: 125 (125.00 diss/flow)
+
+TLS	14	10082	1
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+
+
+	1	TCP 10.10.10.1:443 <-> 192.168.0.1:33063 [proto: 91/TLS][Encrypted][Confidence: DPI][cat: Web/5][6 pkts/6525 bytes <-> 8 pkts/3557 bytes][Goodput ratio: 94/85][0.38 sec][bytes ratio: 0.294 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 40/13 161/59 70/23][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1088/445 2023/1090 747/434][TLSv1.3][JA3S: 907bf3ecef1c987c889946b737b43de8][Cipher: TLS_AES_256_GCM_SHA384][Plen Bins: 0,0,11,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,22,0,0,0,0,0,0,0,0,0,0,33,0,0,0,11]