StrixOS/build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make the script python3.

Browse Source 
Test: Ran the python version of the script and the python3 version
Test: and verified the output is the same. This is for all three
Test: outputs (html, text, xml).
Test: Builds without an error.
Change-Id: I5138b0a1efbbc3ab764c644afc1783c79fd7e250
This commit is contained in:
Christopher Ferris
2021-09-08 13:59:04 -07:00
parent 97e0c2831d
commit 6a54633796
1 changed files with 61 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
tools/generate-notice-files.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
# #
# Copyright (C) 2012 The Android Open Source Project # Copyright (C) 2012 The Android Open Source Project
# #
@@ -30,20 +30,18 @@ import itertools
import os import os
import os.path import os.path
import re import re
import struct
import sys import sys
MD5_BLOCKSIZE = 1024 * 1024 MD5_BLOCKSIZE = 1024 * 1024
HTML_ESCAPE_TABLE = { HTML_ESCAPE_TABLE = {
"&": "&", b"&": b"&",
'"': """, b'"': b""",
"'": "'", b"'": b"'",
">": ">", b">": b">",
"<": "&lt;", b"<": b"&lt;",
} }
def hexify(s):
return ("%02x"*len(s)) % tuple(map(ord, s))
def md5sum(filename): def md5sum(filename):
"""Calculate an MD5 of the file given by FILENAME, """Calculate an MD5 of the file given by FILENAME,
and return hex digest as a string. and return hex digest as a string.
@@ -57,20 +55,26 @@ def md5sum(filename):
break break
sum.update(block) sum.update(block)
f.close() f.close()
return hexify(sum.digest()) return sum.hexdigest()
def html_escape(text): def html_escape(text):
"""Produce entities within text.""" """Produce entities within text."""
return "".join(HTML_ESCAPE_TABLE.get(c,c) for c in text) # Using for i in text doesn't work since i will be an int, not a byte.
# There are multiple ways to solve this, but the most performant way
# to iterate over a byte array is to use unpack. Using the
# for i in range(len(text)) and using that to get a byte using array
# slices is twice as slow as this method.
return b"".join(HTML_ESCAPE_TABLE.get(i,i) for i in struct.unpack(str(len(text)) + 'c', text))
HTML_OUTPUT_CSS=""" HTML_OUTPUT_CSS=b"""
<style type="text/css"> <style type="text/css">
body { padding: 0; font-family: sans-serif; } body { padding: 0; font-family: sans-serif; }
.same-license { background-color: #eeeeee; border-top: 20px solid white; padding: 10px; } .same-license { background-color: #eeeeee; border-top: 20px solid white; padding: 10px; }
.label { font-weight: bold; } .label { font-weight: bold; }
.file-list { margin-left: 1em; color: blue; } .file-list { margin-left: 1em; color: blue; }
</style> </style>
""" """
def combine_notice_files_html(file_hash, input_dirs, output_filename): def combine_notice_files_html(file_hash, input_dirs, output_filename):
@@ -90,13 +94,13 @@ def combine_notice_files_html(file_hash, input_dirs, output_filename):
# Open the output file, and output the header pieces # Open the output file, and output the header pieces
output_file = open(output_filename, "wb") output_file = open(output_filename, "wb")
print >> output_file, "<html><head>" output_file.write(b"<html><head>\n")
print >> output_file, HTML_OUTPUT_CSS output_file.write(HTML_OUTPUT_CSS)
print >> output_file, '</head><body topmargin="0" leftmargin="0" rightmargin="0" bottommargin="0">' output_file.write(b'</head><body topmargin="0" leftmargin="0" rightmargin="0" bottommargin="0">\n')
# Output our table of contents # Output our table of contents
print >> output_file, '<div class="toc">' output_file.write(b'<div class="toc">\n')
print >> output_file, "<ul>" output_file.write(b"<ul>\n")
# Flatten the list of lists into a single list of filenames # Flatten the list of lists into a single list of filenames
sorted_filenames = sorted(itertools.chain.from_iterable(file_hash)) sorted_filenames = sorted(itertools.chain.from_iterable(file_hash))
@@ -104,31 +108,29 @@ def combine_notice_files_html(file_hash, input_dirs, output_filename):
# Print out a nice table of contents # Print out a nice table of contents
for filename in sorted_filenames: for filename in sorted_filenames:
stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename)
print >> output_file, '<li><a href="#id%d">%s</a></li>' % (id_table.get(filename), stripped_filename) output_file.write(('<li><a href="#id%d">%s</a></li>\n' % (id_table.get(filename), stripped_filename)).encode())
print >> output_file, "</ul>" output_file.write(b"</ul>\n")
print >> output_file, "</div><!-- table of contents -->" output_file.write(b"</div><!-- table of contents -->\n")
# Output the individual notice file lists # Output the individual notice file lists
print >>output_file, '<table cellpadding="0" cellspacing="0" border="0">' output_file.write(b'<table cellpadding="0" cellspacing="0" border="0">\n')
for value in file_hash: for value in file_hash:
print >> output_file, '<tr id="id%d"><td class="same-license">' % id_table.get(value[0]) output_file.write(b'<tr id="id%d"><td class="same-license">\n' % id_table.get(value[0]))
print >> output_file, '<div class="label">Notices for file(s):</div>' output_file.write(b'<div class="label">Notices for file(s):</div>\n')
print >> output_file, '<div class="file-list">' output_file.write(b'<div class="file-list">\n')
for filename in value: for filename in value:
print >> output_file, "%s <br/>" % (SRC_DIR_STRIP_RE.sub(r"\1", filename)) output_file.write(("%s <br/>\n" % SRC_DIR_STRIP_RE.sub(r"\1", filename)).encode())
print >> output_file, "</div><!-- file-list -->" output_file.write(b"</div><!-- file-list -->\n")
print >> output_file output_file.write(b"\n")
print >> output_file, '<pre class="license-text">' output_file.write(b'<pre class="license-text">\n')
print >> output_file, html_escape(open(value[0]).read()) with open(value[0], "rb") as notice_file:
print >> output_file, "</pre><!-- license-text -->" output_file.write(html_escape(notice_file.read()))
print >> output_file, "</td></tr><!-- same-license -->" output_file.write(b"\n</pre><!-- license-text -->\n")
print >> output_file output_file.write(b"</td></tr><!-- same-license -->\n\n\n\n")
print >> output_file
print >> output_file
# Finish off the file output # Finish off the file output
print >> output_file, "</table>" output_file.write(b"</table>\n")
print >> output_file, "</body></html>" output_file.write(b"</body></html>\n")
output_file.close() output_file.close()
def combine_notice_files_text(file_hash, input_dirs, output_filename, file_title): def combine_notice_files_text(file_hash, input_dirs, output_filename, file_title):
@@ -136,14 +138,18 @@ def combine_notice_files_text(file_hash, input_dirs, output_filename, file_title
SRC_DIR_STRIP_RE = re.compile("(?:" + "|".join(input_dirs) + ")(/.*).txt") SRC_DIR_STRIP_RE = re.compile("(?:" + "|".join(input_dirs) + ")(/.*).txt")
output_file = open(output_filename, "wb") output_file = open(output_filename, "wb")
print >> output_file, file_title output_file.write(file_title.encode())
output_file.write(b"\n")
for value in file_hash: for value in file_hash:
print >> output_file, "============================================================" output_file.write(b"============================================================\n")
print >> output_file, "Notices for file(s):" output_file.write(b"Notices for file(s):\n")
for filename in value: for filename in value:
print >> output_file, SRC_DIR_STRIP_RE.sub(r"\1", filename) output_file.write(SRC_DIR_STRIP_RE.sub(r"\1", filename).encode())
print >> output_file, "------------------------------------------------------------" output_file.write(b"\n")
print >> output_file, open(value[0]).read() output_file.write(b"------------------------------------------------------------\n")
with open(value[0], "rb") as notice_file:
output_file.write(notice_file.read())
output_file.write(b"\n")
output_file.close() output_file.close()
def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename): def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename):
@@ -154,15 +160,15 @@ def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename):
# Set up a filename to row id table (anchors inside tables don't work in # Set up a filename to row id table (anchors inside tables don't work in
# most browsers, but href's to table row ids do) # most browsers, but href's to table row ids do)
id_table = {} id_table = {}
for file_key in files_with_same_hash.keys(): for file_key, files in files_with_same_hash.items():
for filename in files_with_same_hash[file_key]: for filename in files:
id_table[filename] = file_key id_table[filename] = file_key
# Open the output file, and output the header pieces # Open the output file, and output the header pieces
output_file = open(output_filename, "wb") output_file = open(output_filename, "wb")
print >> output_file, '<?xml version="1.0" encoding="utf-8"?>' output_file.write(b'<?xml version="1.0" encoding="utf-8"?>\n')
print >> output_file, "<licenses>" output_file.write(b"<licenses>\n")
# Flatten the list of lists into a single list of filenames # Flatten the list of lists into a single list of filenames
sorted_filenames = sorted(id_table.keys()) sorted_filenames = sorted(id_table.keys())
@@ -170,10 +176,8 @@ def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename):
# Print out a nice table of contents # Print out a nice table of contents
for filename in sorted_filenames: for filename in sorted_filenames:
stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename)
print >> output_file, '<file-name contentId="%s">%s</file-name>' % (id_table.get(filename), stripped_filename) output_file.write(('<file-name contentId="%s">%s</file-name>\n' % (id_table.get(filename), stripped_filename)).encode())
output_file.write(b"\n\n")
print >> output_file
print >> output_file
processed_file_keys = [] processed_file_keys = []
# Output the individual notice file lists # Output the individual notice file lists
@@ -183,11 +187,13 @@ def combine_notice_files_xml(files_with_same_hash, input_dirs, output_filename):
continue continue
processed_file_keys.append(file_key) processed_file_keys.append(file_key)
print >> output_file, '<file-content contentId="%s"><![CDATA[%s]]></file-content>' % (file_key, html_escape(open(filename).read())) output_file.write(('<file-content contentId="%s"><![CDATA[' % file_key).encode())
print >> output_file with open(filename, "rb") as notice_file:
output_file.write(html_escape(notice_file.read()))
output_file.write(b"]]></file-content>\n\n")
# Finish off the file output # Finish off the file output
print >> output_file, "</licenses>" output_file.write(b"</licenses>\n")
output_file.close() output_file.close()
def get_args(): def get_args():
@@ -254,7 +260,7 @@ def main(argv):
file_md5sum = md5sum(filename) file_md5sum = md5sum(filename)
files_with_same_hash[file_md5sum].append(filename) files_with_same_hash[file_md5sum].append(filename)
filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(files_with_same_hash.keys())] filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(list(files_with_same_hash))]
combine_notice_files_text(filesets, input_dirs, txt_output_file, file_title) combine_notice_files_text(filesets, input_dirs, txt_output_file, file_title)
if html_output_file is not None: if html_output_file is not None: