From 6de4c484ae0de0246e6ee9baf6ac8fe9ac40cac7 Mon Sep 17 00:00:00 2001 From: Grant Pitel Date: Sat, 19 Dec 2015 07:57:13 -0500 Subject: [PATCH 1/2] pyjscompressor was not working on Windows. Function has been modified so that temp files are tracked and deleted at the end. Created override function NamedTemporaryFile to handle different platforms. I also added -a option so that only specific file types will be compressed. --- pyjs/contrib/pyjscompressor.py | 68 ++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/pyjs/contrib/pyjscompressor.py b/pyjs/contrib/pyjscompressor.py index dc9fa362a..3c8f305b6 100644 --- a/pyjs/contrib/pyjscompressor.py +++ b/pyjs/contrib/pyjscompressor.py @@ -32,7 +32,7 @@ import shutil import subprocess import sys -import tempfile +from tempfile import NamedTemporaryFile as _NamedTemporaryFile compiler_path=os.path.join(os.path.dirname(__file__),'compiler.jar') @@ -68,9 +68,10 @@ def compile(js_file, js_output_file, html_file=''): if debug: args.append('--formatting=pretty_print') - error = subprocess.call(args=args, + process = subprocess.Popen(args=args, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT) + output, error = process.communicate() if error: raise Exception(' '.join([ @@ -79,7 +80,7 @@ def compile(js_file, js_output_file, html_file=''): def compress_css(css_file): - css_output_file = tempfile.NamedTemporaryFile() + css_output_file = NamedTemporaryFile() f = open(css_file) css = f.read() css = re.sub(r"\s+([!{};:>+\(\)\],])", r"\1", css) @@ -92,13 +93,13 @@ def compress_css(css_file): def compress_js(js_file): - js_output_file = tempfile.NamedTemporaryFile() + js_output_file = NamedTemporaryFile() compile(js_file, js_output_file.name) return finish_compressors(js_output_file.name, js_file) def compress_html(html_file): - html_output_file = tempfile.NamedTemporaryFile() + html_output_file = NamedTemporaryFile() f = open(html_file) html = f.read() @@ -123,10 +124,10 @@ def script_repl(matchobj): js_output_files = [] for script in scripts: - js_file = tempfile.NamedTemporaryFile() + js_file = NamedTemporaryFile() js_file.write(script) js_file.flush() - js_output_file = tempfile.NamedTemporaryFile() + js_output_file = NamedTemporaryFile() js_output_files.append(js_output_file) compile(js_file.name, js_output_file.name, html_file) @@ -204,7 +205,7 @@ def getsmallpath(path, max_chars): return smallpath -def compress_all(path): +def compress_all(path, exts): # Print headers for progress output print('%45s %s' % ('Files', 'Compression')) @@ -220,7 +221,12 @@ def compress_all(path): files_to_compress = [] for root, dirs, files in os.walk(path): for file in files: - files_to_compress.append(os.path.join(root, file)) + ext = '.' + file.split('.')[1] + if exts: + if ext in exts:# only allow matching extensions + files_to_compress.append(os.path.join(root, file)) + else: + files_to_compress.append(os.path.join(root, file)) if num_procs > 1: proc_pool = multiprocessing.Pool(num_procs) @@ -275,17 +281,19 @@ def compress_all(path): (p_size / 1024., n_size / 1024.) print('%s %s' % (sizes.ljust(51), "%4.1f%%" % compression)) -def dopyjscompressor(directory, c_path=compiler_path, n_procs=0, opts='SIMPLE_OPTIMIZATIONS', dbg=False): +def do_pyjscompressor(directory, c_path=compiler_path, n_procs=0, opts='SIMPLE_OPTIMIZATIONS', dbg=False, allow = None): global compiler_path global num_procs global debug global optimizations + global temp_files compiler_path=c_path num_procs = n_procs optimizations = opts debug = dbg - + temp_files = list() + if not compiler_path: # Not specified on command line # Try environment @@ -315,16 +323,37 @@ def dopyjscompressor(directory, c_path=compiler_path, n_procs=0, opts='SIMPLE_OP except NotImplementedError: print("Could not determine CPU Count. Using One process") num_procs = 1 - + + if allow: + [x.strip() for x in allow.split(',')] + print("Running %d processes" % num_procs) try: - compress_all(directory) + compress_all(directory, allow) except KeyboardInterrupt: print('') print('Compression Aborted') - +# writing to a tempFile has different behavior on windows and linux +def NamedTemporaryFile(): + global temp_files + if sys.platform == "linux" or sys.platform == "linux2": + OPT_DELETE = True + elif sys.platform == "darwin": + OPT_DELETE= True + elif sys.platform == "win32": + OPT_DELETE = False + tempFile = _NamedTemporaryFile(delete = OPT_DELETE) + temp_files.append(tempFile.name) + return tempFile + +def do_cleanup(): + for temp_file in temp_files: + if os.path.exists(temp_file): + print 'remove %s'%temp_file + os.remove(temp_file) + def main(): try: import argparse @@ -352,6 +381,9 @@ def main(): parser.add_argument('-o', metavar='OPTIMIZATION_LEVEL', default='SIMPLE_OPTIMIZATIONS', type=str, dest='optimizations', help='Closure Compiler Optimization levels') + parser.add_argument('-a', metavar='allow', default='.html,.js,.css', type=str, + dest='allow', + help='List of allowed file extensions (include periods separate by comma: e.g., .html,.css) ') options = parser.parse_args() directory = options.directory else: @@ -368,12 +400,16 @@ def main(): parser.add_option('-o', metavar='OPTIMIZATION_LEVEL', default='SIMPLE_OPTIMIZATION', type=str, dest='optimizations', help='Closure Compiler Optimization levels') + parser.add_option('-a', metavar='allow', default=0, type=str, + dest='allow', + help='List of allowed file extensions (include periods separate by comma: e.g., .html,.css) ') options, args = parser.parse_args() if len(args) != 1: parser.error('Please specify the directory to compress') directory = args[0] - dopyjscompressor(directory, options.compiler, options.num_procs, options.optimizations, options.debug) - + do_pyjscompressor(directory, options.compiler, options.num_procs, options.optimizations, options.debug, options.allow) + do_cleanup() + if __name__ == '__main__': main() From 119bdf7e89c54a63a49e4c10fb98d9e2917e65cc Mon Sep 17 00:00:00 2001 From: Grant Pitel Date: Sun, 20 Dec 2015 12:25:42 -0500 Subject: [PATCH 2/2] need to reference last period to get file extensions, some pyjs files contain multiple periods --- pyjs/contrib/pyjscompressor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyjs/contrib/pyjscompressor.py b/pyjs/contrib/pyjscompressor.py index 3c8f305b6..033f855f8 100644 --- a/pyjs/contrib/pyjscompressor.py +++ b/pyjs/contrib/pyjscompressor.py @@ -149,7 +149,7 @@ def finish_compressors(new_path, old_path): def compress(path): try: - ext = os.path.splitext(path)[1] + ext = os.path.splitext(path)[-1] if ext == '.css': return compress_css(path) elif ext == '.js': @@ -221,7 +221,8 @@ def compress_all(path, exts): files_to_compress = [] for root, dirs, files in os.walk(path): for file in files: - ext = '.' + file.split('.')[1] + ext = '.' + file.split('.')[-1] + print ext if exts: if ext in exts:# only allow matching extensions files_to_compress.append(os.path.join(root, file))