How to use zip method in Playwright Python

Best Python code snippet using playwright-python

Run Playwright Python automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

_constants.py

Source: _constants.py Github

copy
1# Licensed to the Apache Software Foundation (ASF) under one
2# or more contributor license agreements.  See the NOTICE file
3# distributed with this work for additional information
4# regarding copyright ownership.  The ASF licenses this file
5# to you under the Apache License, Version 2.0 (the
6# "License"); you may not use this file except in compliance
7# with the License.  You may obtain a copy of the License at
8#
9#   http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing,
12# software distributed under the License is distributed on an
13# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14# KIND, either express or implied.  See the License for the
15# specific language governing permissions and limitations
16# under the License.
17
18# coding: utf-8
19
20"""Read text files and load embeddings."""
21from __future__ import absolute_import
22from __future__ import print_function
23
24UNKNOWN_IDX = 0
25
26APACHE_REPO_URL = 'https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'
27
28GLOVE_PRETRAINED_FILE_SHA1 = \
29    {'glove.42B.300d.zip': 'f8e722b39578f776927465b71b231bae2ae8776a',
30     'glove.6B.zip': 'b64e54f1877d2f735bdd000c1d7d771e25c7dfdc',
31     'glove.840B.300d.zip': '8084fbacc2dee3b1fd1ca4cc534cbfff3519ed0d',
32     'glove.twitter.27B.zip': 'dce69c404025a8312c323197347695e81fd529fc'}
33
34GLOVE_PRETRAINED_ARCHIVE_SHA1 = \
35    {'glove.42B.300d.txt': '876767977d6bd4d947c0f84d44510677bc94612a',
36     'glove.6B.50d.txt': '21bf566a9d27f84d253e0cd4d4be9dcc07976a6d',
37     'glove.6B.100d.txt': '16b1dbfaf35476790bd9df40c83e2dfbd05312f1',
38     'glove.6B.200d.txt': '17d0355ddaa253e298ede39877d1be70f99d9148',
39     'glove.6B.300d.txt': '646443dd885090927f8215ecf7a677e9f703858d',
40     'glove.840B.300d.txt': '294b9f37fa64cce31f9ebb409c266fc379527708',
41     'glove.twitter.27B.25d.txt':
42         '767d80889d8c8a22ae7cd25e09d0650a6ff0a502',
43     'glove.twitter.27B.50d.txt':
44         '9585f4be97e286339bf0112d0d3aa7c15a3e864d',
45     'glove.twitter.27B.100d.txt':
46         '1bbeab8323c72332bd46ada0fc3c99f2faaa8ca8',
47     'glove.twitter.27B.200d.txt':
48         '7921c77a53aa5977b1d9ce3a7c4430cbd9d1207a'}
49
50FAST_TEXT_ARCHIVE_SHA1 = \
51    {'crawl-300d-2M.zip': 'bb40313d15837ceecc1e879bc954e9be04b17c3c',
52     'wiki.aa.zip': '0d85feb259e17d5258f38b2b615a2b87cd628427',
53     'wiki.ab.zip': '7a8c555b9cf3837c9b31c901e9e0142209990365',
54     'wiki.ace.zip': '51555fccbe53b726f6c86a84d704c026a78dd02f',
55     'wiki.ady.zip': '725d2c30c03001c941ac4084549c55c7f8e1d766',
56     'wiki.af.zip': '1a18d34e1b60433b837f5850750a44ca3845323d',
57     'wiki.ak.zip': 'daecc2303cfd05bc6c33b24d78c14e0d7f33e3a7',
58     'wiki.als.zip': '38851192e0b556e566be6c3c93370abf9867e525',
59     'wiki.am.zip': '4576e0121448564b07f448e05e287236343f17c1',
60     'wiki.ang.zip': '9c03da3b06d4becef5d387b9a61438b9362fc36a',
61     'wiki.an.zip': '170f60bdd161cf8e4b5e018acd7d36e8bfc457a6',
62     'wiki.arc.zip': 'c8dad8b00865bf736b087e7b323999ab404bda29',
63     'wiki.ar.zip': '34e9869daa463fdc5609040ff33a03e67512e9fd',
64     'wiki.arz.zip': '2d2790e11e401d46e1bce2970ee5264d5678a32b',
65     'wiki.ast.zip': '1136515e2de556c077324bcd42ffe7f40c8d94c6',
66     'wiki.as.zip': 'f9efde3e4ccda4a1e93fa275a3210f74036e9e46',
67     'wiki.av.zip': '9f8568a3e094a48de4a3b6bea3bdb6fd7e875a08',
68     'wiki.ay.zip': 'f09a422cedc6a0f15fbf30d290febe8057de83db',
69     'wiki.azb.zip': 'd8895581050b9fdb5a10dfec3e27910a150b6faf',
70     'wiki.az.zip': '2a34c2db872597ba3e345ce8b7db138241f9efbf',
71     'wiki.bar.zip': 'd6e40135a6f4ba7a07fab11633034eccb1b05d0a',
72     'wiki.bat_smg.zip': '5d08bd04f0515a36723776c0682b3de0f11d4264',
73     'wiki.ba.zip': '412ac2f3bf9a605e56e2b0990bb0baed41ddf3b0',
74     'wiki.bcl.zip': 'd3717cda357e08390cb57a64e07f5c7b7768d5be',
75     'wiki.be.zip': 'b691e63b8080af23cc37f5f2b21b3154e464c425',
76     'wiki.bg.zip': '08509a510a95e2a8905c19d83faf40d614d2268b',
77     'wiki.bh.zip': 'a812600c6454b779d442b7680e3867e15d895095',
78     'wiki.bi.zip': 'd0d4a3f57419424815f77b3951ef9c7336f6adf5',
79     'wiki.bjn.zip': '0d81879ff7611380896eac6059bb677a5b3fe308',
80     'wiki.bm.zip': 'f3a2a1a8dbc94973a74343c059595a310a66665b',
81     'wiki.bn.zip': 'b3bc70520edf3963c2217873ff5c2537d3545650',
82     'wiki.bo.zip': '2be9fe7701d6a8501461df7bd98fee26859cf83a',
83     'wiki.bpy.zip': 'd44b9267bb4f86e3e43972a6a952cc0ccf90dd3c',
84     'wiki.br.zip': '4bfa66f1ea5aa5cad736eccaa211f6025596bcd6',
85     'wiki.bs.zip': '40c560c5994ab50485d08eeaffd88740f30236ab',
86     'wiki.bug.zip': 'bc7cd87bb067ac477000259cd4f95f45bfb6e4df',
87     'wiki.bxr.zip': '8396fd67ef53f3123540766788a0db54734c4f1a',
88     'wiki.ca.zip': '8f5d3caf0f5d223b2771ec44f7e620e396974fb2',
89     'wiki.cbk_zam.zip': '0af3be50823b564433455d10c8753df88461458f',
90     'wiki.cdo.zip': '19024215aa0c13872c027fc6127b5d7506198b5f',
91     'wiki.ceb.zip': '96374428bf36a43983ba4307d7f6fb5ab52a6c6a',
92     'wiki.ce.zip': 'b27f1a8da448bc9315e15d4261519c64f00de8eb',
93     'wiki.cho.zip': '20944e34c2b58f14adb849dd5a6f5168c7affdea',
94     'wiki.chr.zip': 'b7f41ee3fa76e933e0b5ad6b793c507fc19afe98',
95     'wiki.chy.zip': '4ef66004a609c724fd7d8aab2877f7634323d43f',
96     'wiki.ch.zip': '7f73678b685c9b5f5d6eea9bc00322cfc18d40cb',
97     'wiki.ckb.zip': 'b7db2805526ad8bed878af257b32ca9ba814855f',
98     'wiki.co.zip': '1b9e19b11763cb87ca00520dbdd6ada565547c9c',
99     'wiki.crh.zip': '792003bae25c4471d25721440002c983fa5af020',
100     'wiki.cr.zip': '875e4aa0de8a829e57f6c8e13d43cac5103210de',
101     'wiki.csb.zip': 'fa776014c4c83487d7cb2485bd08eaf6739d9dca',
102     'wiki.cs.zip': 'dca18cb80460522cd281ccc3c9922cf2b3c08b81',
103     'wiki.cu.zip': 'ed23b48ba3193181a358d7a73005afa7655a4fc3',
104     'wiki.cv.zip': '27ccd50942c9c218e00365ee293fa0c3087a7646',
105     'wiki.cy.zip': '78940d5be2969b82c99f785bda2ac5f4e18e149c',
106     'wiki.da.zip': 'a45077d9d73328bd6a96efdba1b31ed9a3639dcd',
107     'wiki.de.zip': '0d9e4bf80100b46237dcb73cfefe390103e7e827',
108     'wiki.diq.zip': '0eef7d9e2f0ce3f100a22dc8fcede9449e466528',
109     'wiki.dsb.zip': '903cd80550931effba1d4e52a19c22592837d11c',
110     'wiki.dv.zip': '3fa06719641ff33ac8a5439d330a8108521da1e7',
111     'wiki.dz.zip': '8bf3937971c3c996493c30b264cb8268627d7bd6',
112     'wiki.ee.zip': 'e66bc50013d884fe69f4f67ba44af2e34fe97927',
113     'wiki.el.zip': '3015f358036658fb126d42fa794d67a90c5b91ad',
114     'wiki.eml.zip': '5be541be6115af5914ac2b8118a09232b771123b',
115     'wiki.en.zip': '7f83d578a31a8168423c77ea25ad381494a5e920',
116     'wiki.eo.zip': 'e7612df98c37cb872f0edc3c3e21dcd2f80a4d69',
117     'wiki.es.zip': '1b7668b23db26810ea433173ce0c11281e801f74',
118     'wiki.et.zip': 'aa31004e7b8ebf359e166b8ea6b8e6f77fac190f',
119     'wiki.eu.zip': '8d7699451cbac4d69750caa8d58b4740cc72e0ca',
120     'wiki.ext.zip': '3aeb4d77c48eb503b26ceb2a76a0a7d841124a71',
121     'wiki.fa.zip': '08b6e805c8623fba526143d46f4685549c4380a6',
122     'wiki.ff.zip': '64f690eda733a6fb4f794e42eb6ff05f09ec1d38',
123     'wiki.fiu_vro.zip': '35c3fdcec0f0dc1ce303212967ea59936641daee',
124     'wiki.fi.zip': '252299a2a59cc0ac07ba25f9458afc26bbac669f',
125     'wiki.fj.zip': '004d1279c27324d02b961341cf0d6ee06dbe8966',
126     'wiki.fo.zip': '12f1d6360d4867cdebcc93be87c024a4709d1af5',
127     'wiki.frp.zip': '8a0f636b5440a9aab38014efada9edfdf94150d5',
128     'wiki.frr.zip': '7c9e7b8109b98aa39b303dd77d837b37e96d4113',
129     'wiki.fr.zip': 'd906e68760153d771e5982009b0150e913254b2d',
130     'wiki.fur.zip': 'd5d2ae08696ed074a581eac563a60eb85467a792',
131     'wiki.fy.zip': '342609d29882fae0a3b402d8ea1478606be0d93b',
132     'wiki.gag.zip': 'f2b91f89dd9b9a1301727476f7823b7260b5f129',
133     'wiki.gan.zip': 'd3ad3c1151555266e1feb9f98b066ee31ee5f410',
134     'wiki.ga.zip': '798b0c26783c7af05d9c4f899ca9fddafeb1e0a1',
135     'wiki.gd.zip': '49085fa182a528bdc51f10e99bef33c88c1e3112',
136     'wiki.glk.zip': '9e16727ffcc691483b69ecbcd331b1df2efa4bcd',
137     'wiki.gl.zip': 'c71c7e6601b2cbdc7930982fbeea636deddd107d',
138     'wiki.gn.zip': '493ccb583211217ccd23e0a43f42ba773bd94f78',
139     'wiki.gom.zip': '45bbd49750ddb7df5afe01fcfd5dda2958934dfa',
140     'wiki.got.zip': '669d018f72827fb965e5ef37e224e21f4682b2e5',
141     'wiki.gu.zip': '4afe874f7830f693e9f83508fc3fb444b33aebdf',
142     'wiki.gv.zip': '9411197eebc07775949d9bb6e440780a68502a5c',
143     'wiki.hak.zip': 'cd1e14bd5d50fa764883b148bda5b821375531e0',
144     'wiki.haw.zip': 'cacd4eb4e476bdd842e8014764b8ae380b346ed2',
145     'wiki.ha.zip': '14acc50950b451f40fe028fd08d042af44732398',
146     'wiki.he.zip': 'a9e2cd13bc2e55d83820c529bac1f518a7198bc0',
147     'wiki.hif.zip': 'dcdd488239deb0ede807cff263ddc972009c21f5',
148     'wiki.hi.zip': '15899ec17985bc0e1db1df497e1b4a51bba1982b',
149     'wiki.ho.zip': 'fde454bb4f3841ea5dde2bbf879138305a4d0b36',
150     'wiki.hr.zip': 'f5d33ba967f7c56538fa9f5f0093f6d634e9db44',
151     'wiki.hsb.zip': '64dc13c7645d2b65b8ba252bd8dfb1c616e8923a',
152     'wiki.ht.zip': 'cf50a5cadcf91aba9ab58d095d65f348e2375d12',
153     'wiki.hu.zip': 'b27f293caedf81a2d09204b11f52a7c8d7443643',
154     'wiki.hy.zip': '641b8666bc2168998989fae1b20a09d3428766bb',
155     'wiki.hz.zip': '1639f9f096de6fac84336a784a391ce73e523d62',
156     'wiki.ia.zip': '37640aaf8a25c02883190951337b5a6f0157d781',
157     'wiki.id.zip': '56ee0c7a38a6d232706932493eaa37b2a87667ee',
158     'wiki.ie.zip': '7c3a5d7f96c801570e2305f45a40d401fcc038b9',
159     'wiki.ig.zip': '405ebc2e8a959163c9f2f8dd015a0bcefd440111',
160     'wiki.ii.zip': '1ec1c7d95d61eeca2dbbd8e432caf88524aaf28e',
161     'wiki.ik.zip': 'e9d088c0d8d0ab420d6d0469c6a0fdb668f1833c',
162     'wiki.ilo.zip': 'cbc9754978ce55e86da2eb3db20579f4a1f19947',
163     'wiki.io.zip': '9e5ab1fd5c4f1094d111f501129e0eecccec69a0',
164     'wiki.is.zip': '0744e63636cf794e0a406c922827628a3dd415b7',
165     'wiki.it.zip': '29f4eb6a5d7dcf45b02b4d08a4a70dfae4c41200',
166     'wiki.iu.zip': 'fb2e8de825d554257768d363a3a09f711afb001b',
167     'wiki.jam.zip': '077cfb6de9d025aee4a5b2ea9ce15ada02f10a4f',
168     'wiki.ja.zip': '7940f6c2bc490c04902f0faf0562b92cae7136bf',
169     'wiki.jbo.zip': '3d086b6c9a369f197516cd0dc699a94612f45c6a',
170     'wiki.jv.zip': '2f68cb3436b27a25ddfa40fab3e2cd44574b437e',
171     'wiki.kaa.zip': '9fd5df362b7cb615f2267084d8b3fb8608be2693',
172     'wiki.kab.zip': '96abf1440ad21de58d7274d3a16885ef4a2efda4',
173     'wiki.ka.zip': '72ddb2382c87184fc05a93e89ed8aa4f54a62a0a',
174     'wiki.kbd.zip': '81dfc3c6f8581c2aa15342c84688b4ba59b81cc6',
175     'wiki.kg.zip': '4d07cabef6f804fc6432d3f630675ed4cbbdd49e',
176     'wiki.ki.zip': '59b5c31df227ff9454ad8b3a1d16b065620dbddf',
177     'wiki.kj.zip': '751b80c4a4d82dd217d3d2b3905eb39b349874d7',
178     'wiki.kk.zip': '7fb733a2405f421a7c49b756381a52965a8af205',
179     'wiki.kl.zip': '05a9d5c9bf12d8845356f88b546418d2e40f79c6',
180     'wiki.km.zip': 'da0a67028fa0244a2e7257ae259c2f7a7544dc66',
181     'wiki.kn.zip': '6cead946350b31fb2f353085fd00b8ea9c9ecc77',
182     'wiki.koi.zip': '0c61f83434404267527eaf583e89b4d8bb3a6a65',
183     'wiki.ko.zip': 'c0825282faf1e7af6820bd8b28d06c77760dcbe4',
184     'wiki.krc.zip': '0df3c3f0f89521299dab741be3d698b2c94c194e',
185     'wiki.kr.zip': '71651f046cef420fb28ca15e35720bb7747c4586',
186     'wiki.ksh.zip': '8b9ab88baa49e72e40a5a80bef98f3ea2afbdd07',
187     'wiki.ks.zip': '02af37f12753662c9e7bcac3b8786dfd2f298710',
188     'wiki.ku.zip': 'ca1d370b327ceca025884bf83139456024a3a978',
189     'wiki.kv.zip': '28b3617c5566f3182f14bf11a906456b227840ba',
190     'wiki.kw.zip': '075a02e8eaae26897c23898fb4d36f4e41e4d1d0',
191     'wiki.ky.zip': '771601a934cd4d0a98e5059f6389d2496e8dcf7c',
192     'wiki.lad.zip': '2788ba3f275d72299e877c96cde106bd8590f405',
193     'wiki.la.zip': '759f6365874442ab8e04d992b047f53ad74231a6',
194     'wiki.lbe.zip': 'c8105f1cf8a3d46ccfacff1d40a581f442b3c4a1',
195     'wiki.lb.zip': 'dac5af52364f2c0d3a0c794411465d1254f2fb48',
196     'wiki.lez.zip': '17331cb779dee8cb60f2734213af80d57acfcfad',
197     'wiki.lg.zip': 'fd4e2d67d1f098474053abc9a1984dfe4a2854b7',
198     'wiki.lij.zip': 'c29157f5e4d2b37c01cf6e389f03ddafef6acdb2',
199     'wiki.li.zip': '10490e49a12230af2127543da69c427f92c6508f',
200     'wiki.lmo.zip': 'cc44163572deddd78af6b006394f623cb21934fc',
201     'wiki.ln.zip': 'bf52699c5cbf79bedb2e2856d8a720189b6864f3',
202     'wiki.lo.zip': '3fd8a70d8e26071a365f10016875a4a4f15ffcee',
203     'wiki.lrc.zip': 'e262b4fcc55cba48d997cd06d006b82a5abe09a9',
204     'wiki.ltg.zip': 'df6a83f2fab35f9a2f97fd8d857cb1cfa59f331f',
205     'wiki.lt.zip': 'a738a3f29a6a5481082a7a9a41b2040b9cf537e4',
206     'wiki.lv.zip': '8e328d99aacaa021fcc51425caebc063e22e6cf4',
207     'wiki.mai.zip': 'e909de86c27eced2cb5f02f550da7fc2502b5eda',
208     'wiki.map_bms.zip': '192bf6b88f955746abb398893868482730585e3a',
209     'wiki.mdf.zip': '3d0d5da3c85bef8ae52f0fd17e314a1960a26d36',
210     'wiki.mg.zip': 'fe66055b63ce8771bf43f8dd543bbd967f8ea8b3',
211     'wiki.mhr.zip': '33514c98da3bd9602851db96fa3dd8192aac0674',
212     'wiki.mh.zip': 'dc77309103c6cfed7ff095b3f9f158e1ae437e71',
213     'wiki.min.zip': '8b925eea6df0411ee09baef5801d807cfec8cfa4',
214     'wiki.mi.zip': 'd57831e8d7cb2ec260fc9d83d4281f0bacfb29a5',
215     'wiki.mk.zip': 'b1fc2d85527e99530a93e3bbc5fa9fcde89910f3',
216     'wiki.ml.zip': 'b9d53b8e76a05f5e959afd190da3015b36793297',
217     'wiki.mn.zip': '715bf0ee67b48ec872659380fcf63ad006ddcc7e',
218     'wiki.mo.zip': 'fb273fe373eb61310051d94ad6911320f573d0ec',
219     'wiki.mrj.zip': 'b0d1e43e37e1718c8e05fd81a511095636def361',
220     'wiki.mr.zip': '67e942a7742cc957298c8cd0cd0af0531dc936d7',
221     'wiki.ms.zip': 'e218f113702b039fc8e80a77b894cd9fa4eff77d',
222     'wiki.mt.zip': 'd68d5b636eac07b2e1307186c2c05b9a80e39658',
223     'wiki.multi.ar.zip': '31c7b742c63c3367e9bce5c4dca37d5ceb33f1a6',
224     'wiki.multi.bg.zip': '8991e8123bce7fd6c8e4510c71ede5715ae36f01',
225     'wiki.multi.ca.zip': '0786e071438150485d394a4bf2e976d3a1b313ff',
226     'wiki.multi.cs.zip': '7237f291146e69f0fc7002a0e175c7fd003d44e8',
227     'wiki.multi.da.zip': '5591c20015191101aee190c02738c99073a8fe76',
228     'wiki.multi.de.zip': '986160e51a08f4a93f1573d17352e375cbaedd6d',
229     'wiki.multi.el.zip': '570eb12811ce61f6176f263eff3e945be69e7da0',
230     'wiki.multi.en.zip': '2c3ef35d8338d4a905e7d10645572ab7a6730d44',
231     'wiki.multi.es.zip': 'c1db7c7175665a7230f92ed038b78de780e060e9',
232     'wiki.multi.et.zip': '54d0515865c754331b445dd9ba0ae7ed79b770aa',
233     'wiki.multi.fi.zip': 'c94abc803a42b89cd75b278114b1f2cf4e2f3ecd',
234     'wiki.multi.fr.zip': 'd4904b79eaf8ae386a7011ad84afc9b4238c9928',
235     'wiki.multi.he.zip': '370ec2a379eecc2d2e984cde3e0f6d0a027eade7',
236     'wiki.multi.hr.zip': 'd3f25ae76b040ffa09e964f6edc55488f6086394',
237     'wiki.multi.hu.zip': '4b64bcdf0fc1f01bbd8427bd7bf6b46319308e7a',
238     'wiki.multi.id.zip': '3ad5f590d5c847b35a334f1bdb48b9c466f5de68',
239     'wiki.multi.it.zip': '18746450e665e96c33f2e2026986f643a27e0945',
240     'wiki.multi.mk.zip': '1d899f1449d8729b7dbae226f05151a656694626',
241     'wiki.multi.nl.zip': 'ff0a04dbb07c2cdbc61d5a241175e30ed46b48d4',
242     'wiki.multi.no.zip': 'd1af729024181e64f58ae37ab233fc53811e2601',
243     'wiki.multi.pl.zip': '91c3984c4f3158b1cb1ff11d8cc4f9240631266e',
244     'wiki.multi.pt.zip': 'a1782c4fa4337008f82c0e2bf78e4323d145be29',
245     'wiki.multi.ro.zip': 'b1a0840d084009ce00c47a3c24c984648dbe8785',
246     'wiki.multi.ru.zip': '540607ba4334dab6089de463f974861aac8a35ae',
247     'wiki.multi.sk.zip': '2a2bb39e011cf2bf6dcb8cb6c482b8eb9764eea3',
248     'wiki.multi.sl.zip': '99442dab442dc196c107868db9174c78e270db1e',
249     'wiki.multi.sv.zip': 'b40be83d2d7c27633c712aea62ceec0d409cc03a',
250     'wiki.multi.tr.zip': 'e2bffab1616f54d180ba3d8bfe5e94ec9a489184',
251     'wiki.multi.uk.zip': 'e97f64d9ba2b58a5e80c9b896b87340aba1e0eb0',
252     'wiki.multi.vi.zip': '532fa24d8787a8906fb04a88e74a713b00cb33ec',
253     'wiki.mus.zip': '1bb0cad10889b8a3bfa36c36c7da1f2fb2237bb8',
254     'wiki.mwl.zip': 'e3d1fd1fa6290521d403e84eba577e552e330844',
255     'wiki.myv.zip': '64a6505691441778766b7941b5e7f45a624a64a5',
256     'wiki.my.zip': '491ce8dbf174d4abff758db4950f49eda90883d9',
257     'wiki.mzn.zip': '76abf410749fd4516ead20ced891b54245fcd4a3',
258     'wiki.nah.zip': '0496592cdd70eaf61b257fb5345843d38f425592',
259     'wiki.nap.zip': 'f0df66cdbef5734f0afeb806cda631722fb426d8',
260     'wiki.na.zip': '2456e4776b5e985cfaedfac244e0b40cff4e613c',
261     'wiki.nds_nl.zip': 'ffd10e05b749281634eb7a758102d8d6ff42760e',
262     'wiki.nds.zip': '2455e9fa4294828b25b32bdad7307a105f9fbe1d',
263     'wiki-news-300d-1M-subword.zip': '697f4c8f37443be3aee7b96abe28fd7ebec95ef3',
264     'wiki-news-300d-1M.zip': '567ef9c2e207be25da23e61312e6ba620da30466',
265     'wiki.new.zip': 'a781885678cc1079d4be221c414339eb9bee8d19',
266     'wiki.ne.zip': '180b068343288cda40d012aaa99d29459d341eb4',
267     'wiki.ng.zip': '6db8111ab700f7b0841af87f1f1453341048014e',
268     'wiki.nl.zip': '582420f290947cf38503b7f4b8ea9bb21918005e',
269     'wiki.nn.zip': '4a0e30376b361ee19800e6d897a865572e330f84',
270     'wiki.nov.zip': 'ac98c0300302019ff855698561708abd81730db3',
271     'wiki.no.zip': '6893a7912ab3756e31d09ef1f9023c27c0b047f8',
272     'wiki.nrm.zip': 'bd27aadf25a165ebbac486437ea6a06b710fdda6',
273     'wiki.nso.zip': 'c55dfebb83351c952831db34e779e0a380212f05',
274     'wiki.nv.zip': 'cf122e5ee041287917c594a2cb6cd247978f1ec0',
275     'wiki.ny.zip': '9086021a60babd7e87afa469dbadb004523f5fd2',
276     'wiki.oc.zip': '15075544cf837135127d8688cd06fb8e4c8b7f3d',
277     'wiki.olo.zip': '523628bb652e1563b4dd5a94b518addf10699f74',
278     'wiki.om.zip': 'a29360ab3930d889c4eb5b385589f84c1ff9f06e',
279     'wiki.or.zip': 'a782e649ae5307dece445b0c11b15ffb9ce88297',
280     'wiki.os.zip': '0d76ca005afd48b87dea5c9784c4c48bb51d3e3e',
281     'wiki.pag.zip': 'b046ef71badc9d7eec161e3aec2ffc3abb7bad20',
282     'wiki.pam.zip': 'abed25ef407e05209f2653d571bba5bc7c66e7b3',
283     'wiki.pap.zip': '5d099bfc65c85f824634a191ce33e8e42f947ded',
284     'wiki.pa.zip': '2066ed0016720b9f8779f55f2cc2de08511025f6',
285     'wiki.pcd.zip': '66914c99e5531c0484448b84568971362cdad0f6',
286     'wiki.pdc.zip': '6ed181fa1f8782917ae7849490c0a5cb0b0b9b29',
287     'wiki.pfl.zip': '8d271226af8509962b15a96c4d6e41d9aabd972c',
288     'wiki.pih.zip': '365955dbecb17027435fe487ab92a7a267fa25bd',
289     'wiki.pi.zip': 'eeb863545392c92cff0f3e3d9c3f61539d3fa1dd',
290     'wiki.pl.zip': '2b0cae8af2637bc24b958e6757149d1b9f8c8fea',
291     'wiki.pms.zip': '9eff2e96e1cb9bf02adf816c4feb5aa3cd1a384f',
292     'wiki.pnb.zip': '23f77d1d9469f5b2c342984288cb3092d53d8dee',
293     'wiki.pnt.zip': '84cc9532d2fd7b322bcba91e01ac36c9a719e23a',
294     'wiki.ps.zip': '18c9ffb2a81cbc25299b26e35170a29b7de9309c',
295     'wiki.pt.zip': '37752109a44829de5ea10b173d7c0cecc0b1a0d7',
296     'wiki.qu.zip': '5582c07eeeaec10d9382b3ab90d2921fc97fa2e0',
297     'wiki.rmy.zip': 'a106ab536001e92e7a9708417faee9418f4058d0',
298     'wiki.rm.zip': '67a324941f2b895a418fbd89314a18bfda19b1de',
299     'wiki.rn.zip': 'ce17294909c046e90bb0131632e1d795d1771816',
300     'wiki.roa_rup.zip': 'a9a378e90cd46353283c92cfb7d34dd485a018d2',
301     'wiki.roa_tara.zip': '953fe4cf1667cbb9b3b8e11666885bfedf74b411',
302     'wiki.ro.zip': '6bbb0f9452398416d9183e00e6cd091a02fb351f',
303     'wiki.rue.zip': 'e9f9b8ab63c7722b4b68e8c465b1c69436132553',
304     'wiki.ru.zip': 'f8f68aa5792941d7750b545e56f1ff5127e88cc2',
305     'wiki.rw.zip': '018b9fb76fca5ce7a3e1f266df33fcc1bbc50493',
306     'wiki.sah.zip': 'f6c94dbd3b719b154217388310fab72e5a69f823',
307     'wiki.sa.zip': '4dc78b48d651056546d14b659c6598770c6bce77',
308     'wiki.scn.zip': '218ba35c042cb3e179988bac9acf51cccf37422b',
309     'wiki.sco.zip': 'daa8cedbb223e87d48f720aed9ce63dd0c81c632',
310     'wiki.sc.zip': '909cc5160cad60fda34ab89c2b87ae4229402eeb',
311     'wiki.sd.zip': '5468ed141bf2f1d9b1f8d7b31fee926b496ea9db',
312     'wiki.se.zip': '0eb962f8768d88ffcbde3aac833e134a263c2055',
313     'wiki.sg.zip': '651035aa74dc2f515253444f48aa9911094f9d27',
314     'wiki.sh.zip': 'cf3057b61bd5bca6f47640801681d451aee210cf',
315     'wiki.simple.zip': '367737535e39defb0e713a7ff2374cb932c5a9bc',
316     'wiki.si.zip': 'cebb2f4011b0d679fe856c5950076e3c48496ecc',
317     'wiki.sk.zip': '6c43758d0c0f52351210c558cc33266a65709068',
318     'wiki.sl.zip': 'd0239eefc830e5919bef8d9173a884e9e7371e7a',
319     'wiki.sm.zip': '2e3cf33f17b449c8f81cc9ea4c84d542cfd23a14',
320     'wiki.sn.zip': '4d3844ee350ee0065e5fe910a3f669ef863a2fc9',
321     'wiki.so.zip': '9da45db9b21d1f27c4f73152539c1e4fc9b1c49c',
322     'wiki.sq.zip': '0db976ec147df49e648cf8256562371d0ae6f2f0',
323     'wiki.srn.zip': '120e229d522cc22008c50e0eb74b23d9f6eca51d',
324     'wiki.sr.zip': '63b67391158bdd7a642f7d8412771c22e1041744',
325     'wiki.ss.zip': '4368f7931f6730a6e8cb9b5794906f2d827582a8',
326     'wiki.stq.zip': 'fb1ba577bf6fb7f7fcdc52bf392e63ed8492465d',
327     'wiki.st.zip': 'b7e96392b3880c19e210fd42bc72e3f76c07a4c3',
328     'wiki.su.zip': '4c4880cfca1ff954c88e44a32f201218eb2be146',
329     'wiki.sv.zip': 'e2b10091585f795dd18289c4a65a1da591a78196',
330     'wiki.sw.zip': '726631d8998ba1647d040e6b70f4bad7b8d8c367',
331     'wiki.szl.zip': 'a70de974cff95cad0443f5faa6c8412c92998100',
332     'wiki.ta.zip': '6bafd0bb523f654038393ba191012527745b940b',
333     'wiki.tcy.zip': 'b4bd573eaf9fd87300a25648b38a053161d12c39',
334     'wiki.tet.zip': '7e5608958977164e544850a5a169f5d55cd47a20',
335     'wiki.te.zip': '948e5a6ec13ac95b595c3f52a6e7b9642a56c530',
336     'wiki.tg.zip': '5b46429024d6819f6b511a4924b90c958615d40e',
337     'wiki.th.zip': 'b8ee0878cec41b4ab1055a17d0ed669de1ed9afd',
338     'wiki.ti.zip': 'd55abb74bb3ff195d2293ee9e77886111ee50e52',
339     'wiki.tk.zip': '20263f39a31a1d55343f9dea7aecaa2860aefde8',
340     'wiki.tl.zip': '2f2b809017249f8c4f8d5eb62979b58f16e8732b',
341     'wiki.tn.zip': '0aa11b07b1ad6437bc1e9b6476d51ddd35dad994',
342     'wiki.to.zip': '6b90b32ae258a56e67b42736675236b91163b3ad',
343     'wiki.tpi.zip': 'ca9591e621ae667a1521d0bb5275435d45e974cc',
344     'wiki.tr.zip': '3b6f86c2a115c7adec1b073b1f5624890e680148',
345     'wiki.ts.zip': '8a00b16f2881977ad6f8c8665316c27fcab9b842',
346     'wiki.tt.zip': '8d2f559bf1e09180d6dc4b127d61815a27670a20',
347     'wiki.tum.zip': '5b3f6f3d8cae4d9534cd1fd3afc2f64ec8342b8d',
348     'wiki.tw.zip': '7c189fabfcdb2973178c25d35fd10e46ee7148aa',
349     'wiki.tyv.zip': '5e3811a19bbf961a5361ac37ff3502287c9ab022',
350     'wiki.ty.zip': 'a7f31f8cabf4282533773aa7e63f294315cc85ea',
351     'wiki.udm.zip': '643df5ab0914535e46e6839845d0ab585c81a119',
352     'wiki.ug.zip': 'a5388269893ac4c7da28b2284f3536ca0f3c9341',
353     'wiki.uk.zip': 'fdc9b0a0ab806e5845e9d89b8887ec9d555a0547',
354     'wiki.ur.zip': '75579eb5609ea31d79bc2d1bd81d01f48e01bc7c',
355     'wiki.uz.zip': 'aa149200f8c6e3e8bb5aa3c67112675d136900b8',
356     'wiki.vec.zip': '58c4c9528154e256fbefeb97b8c1675356079f74',
357     'wiki.vep.zip': '966b371afcc383058a5fbc6ee8f822620f03feac',
358     'wiki.ve.zip': '6450e3ec2c78980c5a41d71ff159aa27918dda75',
359     'wiki.vi.zip': 'bfa287fbb358a66b4f9576585df3e46607e1595c',
360     'wiki.vls.zip': '7335bfda43890f42e045b8a5de25d1a8629fe012',
361     'wiki.vo.zip': 'c2ca18bea165cb1253c1d88fa9958a25088fc84b',
362     'wiki.war.zip': '5cda8fdd64e3acf5488ad361b68a63fb23747559',
363     'wiki.wa.zip': '2e538c10a0e9f43ea5875c90a8ce01a07c4695a7',
364     'wiki.wo.zip': 'f54c65ab63f98ffec7b3fb5bdd51a814034bd673',
365     'wiki.wuu.zip': '68d9ad802836737392d62056231bf1b7a58594c9',
366     'wiki.xal.zip': 'fb39fed41ccba2e4e58ab7714a53aae3695dbe04',
367     'wiki.xh.zip': 'd37caa4d94e66588879231d0826798d8aa4b0a44',
368     'wiki.xmf.zip': '956c43bca0d88e9348099cde43d58898e43d9f27',
369     'wiki.yi.zip': '151c1670c48e976e4202272b066d7080a8c83615',
370     'wiki.yo.zip': 'fdbd0fc6e35bb04c3aef1fa6f0262ba261b11199',
371     'wiki.za.zip': '11f6a5dcb49c4d0571d5ac4fb3d7dda1d378fc06',
372     'wiki.zea.zip': '22159a722c5c0390bad9206eb75e6e166efe38e9',
373     'wiki.zh_classical.zip': 'c689d61d2254caf1ecec0909249523b09a737717',
374     'wiki.zh_min_nan.zip': '0516a413565484d924a4c8b50c690d39344cdb64',
375     'wiki.zh_yue.zip': '464f4c1c2039194cbae7502ed3a2eeff4df9e34f',
376     'wiki.zh.zip': '2374ec566f6411b9bb570077636695fe9768a5ba',
377     'wiki.zu.zip': 'a6d0325dab37cd551e6d7f6c783dd13f4c71db2f'}
378
379FAST_TEXT_FILE_SHA1 = \
380    {'crawl-300d-2M.vec': '9b556504d099a6c01f3dd76b88775d02cb2f1946',
381     'wiki.aa.vec': '5cce30fc85471572c498f278bbe495184577363e',
382     'wiki.ab.vec': '9d89a403a9a866d3da8dd8cfab849f59ee499343',
383     'wiki.ace.vec': '85d00074f7a08626f39da6a0c8a5cfa250096ab9',
384     'wiki.ady.vec': '9d17d74f0348224cdebf8a831e61af0825f8952d',
385     'wiki.af.vec': '999e64bcd8dab8de42cb1feceeca360def35324d',
386     'wiki.ak.vec': '6092b8af335c2dc93e8df2bbf1d715f01e637bb4',
387     'wiki.als.vec': '96052e96870695cca50857b5fde5f9f42219139a',
388     'wiki.am.vec': 'dff7fcdd8f5ba0638ab9e1758a89800766156d72',
389     'wiki.ang.vec': 'a7c30e02422d97d23a0701279c5c1c03159130a5',
390     'wiki.an.vec': '5b4c2b1de5c04e4e0be83841410ca84c47305d21',
391     'wiki.arc.vec': 'fd3ad743103f80cde9cfc048d7ca509e50efb35a',
392     'wiki.ar.vec': 'c46e2142f799cc385bd25f0c0a8943ca565505a4',
393     'wiki.arz.vec': '5e904087043b91f4945dd708f4230fdf51360132',
394     'wiki.ast.vec': '89a90357101953b7c292697fd050c00fe5c38ac5',
395     'wiki.as.vec': 'cad5883b5147cbe6cdbf604f65cabdb675a59258',
396     'wiki.av.vec': '99976a63ca8c4231f808fd4314f0433db35e290d',
397     'wiki.ay.vec': 'be359dad25b2c742d3abfa94c5f5db13f86c730e',
398     'wiki.azb.vec': 'e23af0a436b97434813c3cb14ed114cc5b352faa',
399     'wiki.az.vec': '9581d55d9056ad398a153c37b502f3a07867d091',
400     'wiki.bar.vec': '96130f1f2e5bffdd06c202ad4472e5234020980a',
401     'wiki.bat_smg.vec': 'cb3aef58da2011183b39fca64cabf3d9d7a62f4b',
402     'wiki.ba.vec': '22147ee16b2d163cc88d09a035264fd0c10dab68',
403     'wiki.bcl.vec': 'd4117b5c443438ddfa608b10a5be2c2501817e7e',
404     'wiki.be.vec': '6cf81322cd7b046a7f02ec4c4960ad27045383fa',
405     'wiki.bg.vec': '7c1cc6d0c52b038e4b7173259b0c009f242cf486',
406     'wiki.bh.vec': 'ab2d29017afa015c49566a6d9bf75393c23ac4c0',
407     'wiki.bi.vec': '15785220cd6e6c86cc87e7d3f3322a5541a4fe5d',
408     'wiki.bjn.vec': '5f134cf288e8042dcd048a3ee76159aab42c7288',
409     'wiki.bm.vec': 'f36a19c95e90865f6518d4487e59f363b47bd865',
410     'wiki.bn.vec': '6fc3bfd9af455719f55bee0bea31b11afc70cf06',
411     'wiki.bo.vec': '2e9358e03dcfa09da23d2e1499d84b10348fd8a9',
412     'wiki.bpy.vec': 'c2bb15487c4bdb8fa869772694300ae1fee73896',
413     'wiki.br.vec': 'df44e16abd2017e2a1b6c6588ee02779b19907f6',
414     'wiki.bs.vec': 'c4943a290819ceae1611dd11179b40aab0df0471',
415     'wiki.bug.vec': '942d8f7dadde5faa33aa72862501434f48e29f60',
416     'wiki.bxr.vec': 'eaf767690c6b194605ae778719212e3874873d4c',
417     'wiki.ca.vec': 'f5971edee11c939f6a7accfd33a9a45caa54141a',
418     'wiki.cbk_zam.vec': '6fef47b4559eec402ce371de20dfb018acd6347d',
419     'wiki.cdo.vec': '95e8196bf76323dbabab1b8a49ba4d677af3ccea',
420     'wiki.ceb.vec': 'b8516a55537b8f80c927d77d95cdf7e4ff849a05',
421     'wiki.ce.vec': '1d94b0168a773895b23889f7f07d7cf56c11a360',
422     'wiki.cho.vec': 'cec6778f025fa9ae4134046c6c3a6291bd9c63f9',
423     'wiki.chr.vec': '8501bf86b41074ed6c8d15b9209ef7ce83122e70',
424     'wiki.ch.vec': '46803f3a1734f6a7b0d8cb053bbb86a6915d02e9',
425     'wiki.chy.vec': '26c87688551ffe3a0c7a5952e894306651e62131',
426     'wiki.ckb.vec': 'adb2fef309f1d93f429442b9c16c1564192c58f3',
427     'wiki.co.vec': 'af876a918594e5541207bc12f17bfc4268df7b93',
428     'wiki.crh.vec': 'c0d2310a1207fcacc94b25b149420b33bf835015',
429     'wiki.cr.vec': '61dd9f044b7dfa56dcf1c3c07c7504c569420528',
430     'wiki.csb.vec': '649cb2692f08414987c875dc331022567d367497',
431     'wiki.cs.vec': 'f3ec1502aeee6a550d8cf784273fa62f61419a4e',
432     'wiki.cu.vec': 'ddadb14ea00ea1dda716ee33732497ec049b526f',
433     'wiki.cv.vec': '9cdb0bee5a0fea030def85597dba7108f21b0424',
434     'wiki.cy.vec': '32d976a9bfc4dd6e39328c906eead0f597bd9e25',
435     'wiki.da.vec': '526947dab1ffbc1465c7a766f2bca4de50676b08',
436     'wiki.de.vec': '2ed2696afe55f023b0040b238d9a47e5fedfe48b',
437     'wiki.diq.vec': '77f3c370d1d77806fafe368cf788af550ff607dd',
438     'wiki.dsb.vec': 'e49a647a441fbf011ac5411dd6005e8725b9a65d',
439     'wiki.dv.vec': 'e135ba97c711a021bc3317db2b95db5212c17658',
440     'wiki.dz.vec': '24888f0b2cd156360bfb5e9e905240163ba798d8',
441     'wiki.ee.vec': 'afd1670655daa7ffba51187a415fdd0b43f1d487',
442     'wiki.el.vec': '6f034271390feaa6f9d7d16f933ddef637755979',
443     'wiki.eml.vec': 'de6be7a2ffdda226eec730dd54b4c614bd7f5dca',
444     'wiki.en.vec': 'c1e418f144ceb332b4328d27addf508731fa87df',
445     'wiki.eo.vec': 'b56998fd69f66755b722a9481a9bdaf10f62c9aa',
446     'wiki.es.vec': '2f41401aa0925167176bcd7a6770423d891dfef5',
447     'wiki.et.vec': '64d56b66c02d5e49b1b66a85854d67d2dd9ebd41',
448     'wiki.eu.vec': '5e72f4ef93666971fea5d2180b354e0a0821ba91',
449     'wiki.ext.vec': '456c5632b13a0f136cd180ebe2dda67b83f78397',
450     'wiki.fa.vec': '09b6cc685c895c66b853af9617787d3ab0891e2c',
451     'wiki.ff.vec': '12b09d695f5fb8de4b5da9d36a73eb178b293a04',
452     'wiki.fiu_vro.vec': '168a71a2b1c478e6810fa5dce9612d8bf8a273dc',
453     'wiki.fi.vec': '91d19baae994d7e556b5b5938be2dc6013f9c706',
454     'wiki.fj.vec': '36d36dc14001a109926bfc633594f6a2f7401697',
455     'wiki.fo.vec': 'eead8ddc7bb74b12b16784723abf802bb51f844d',
456     'wiki.frp.vec': '0eb70a613ccf807c7308c1f62535f0606465029d',
457     'wiki.frr.vec': 'cde62af939cb2de35e341cef2c74813802a58ed4',
458     'wiki.fr.vec': 'b092229005a65d8683a4112852fe6eb8161a6917',
459     'wiki.fur.vec': 'd4a595cffa1abcdcf4229ba15277179ce5d20bc6',
460     'wiki.fy.vec': 'd4beef537b7ff142a3986513879ff51a9ec14a7b',
461     'wiki.gag.vec': 'c82ec7a5d081f0673661824f4fc34345dee255f0',
462     'wiki.gan.vec': '7e53a33b7bd5b0360ea4cb452145616c09445029',
463     'wiki.ga.vec': 'caaa5b2167a499893313ac1aa38416a6a0fe9a24',
464     'wiki.gd.vec': 'f4b513598a1bf0f0d5b6521ea8ce363e9596cb97',
465     'wiki.glk.vec': '20a7759075916e10531f5b3577302353cef565cd',
466     'wiki.gl.vec': '8888bb8f3d70b36729b9ae479fe3765e0c083862',
467     'wiki.gn.vec': '98594af7897c5a1f35885ddecc77556a7e7ae981',
468     'wiki.gom.vec': '5a1193d9e5d49d06354c14e2b7c01bea176e13f1',
469     'wiki.got.vec': 'dfa06de83a0e3099027c57b84561d7d990ea8310',
470     'wiki.gu.vec': 'f9e13452eb63d92bea44c7c3db8fba9945c7000e',
471     'wiki.gv.vec': '993a7ee31bdacc91763dad656aa6c2947b873473',
472     'wiki.hak.vec': '9e83512d34c7f81739492bf0abbb25ff1ef88573',
473     'wiki.ha.vec': '677a24efeeb1bcb8c0a931407775f18b18e875ae',
474     'wiki.haw.vec': '58fea5aa1b37723797d26fb3d050ce6176757240',
475     'wiki.he.vec': '55534560247394669e3f5c169136770c93bc2708',
476     'wiki.hif.vec': '49697cf784814d3f1a47559724028e0fc0940d36',
477     'wiki.hi.vec': '8049bb8604bc049d48bd934e27b0e184c480a413',
478     'wiki.ho.vec': '9c75a09e099213aa8cd1f1020b223427537cbdd8',
479     'wiki.hr.vec': '0c96f9af092cf8a84b03aec1426cd23921671489',
480     'wiki.hsb.vec': '3dc7830544c58535bed308c552d609e13b973502',
481     'wiki.ht.vec': '5039dfb58a074ac046813f2dae81159be8c5213f',
482     'wiki.hu.vec': 'cd777e9efca3d4bd97c89f01690cfa4840d9c46f',
483     'wiki.hy.vec': '21f9259d04cfd22db446a45d3622af225f00cf20',
484     'wiki.hz.vec': '2a94b1390d68027748a05169fbc0c11a9a183456',
485     'wiki.ia.vec': '2a348dc924638efc20c34785852b0837364aed76',
486     'wiki.id.vec': 'c49d5c9bec89114599427f6c12a5bda2e5523dfd',
487     'wiki.ie.vec': '01b0d11c0e7397418e73853d220e97bdcf7a8961',
488     'wiki.ig.vec': 'd2d1643b4fb1a18a4d002cf2969073f7f201b3b2',
489     'wiki.ii.vec': '41c6cd68b3ebe4ece2a06c37b06dca5d07c9fb3a',
490     'wiki.ik.vec': 'af31cbec7b839f50fa70553ec63c58f7067d3ea8',
491     'wiki.ilo.vec': 'c0e43835a3f4e0033ea5d7c6ff189982b2f26a05',
492     'wiki.io.vec': 'af0c480c5872bff31d82e767c1116da2a6be0c00',
493     'wiki.is.vec': 'ae0b018f92b3e218f2dacb2045a8f0a0446788a5',
494     'wiki.it.vec': 'ac4a985e85ffae48047034e2603d804bf126caa9',
495     'wiki.iu.vec': '5d51b2ba215005216ae003f4a6d6ef39fb30ca2e',
496     'wiki.jam.vec': '6d51e384c56330097c2531fdbf4e74418909e388',
497     'wiki.ja.vec': '7a2b1af1e46d795410692a002e40fa3085135f69',
498     'wiki.jbo.vec': 'c90481946aa4b6b304528292612ae620f6549f3e',
499     'wiki.jv.vec': '2ff7927d3ff04b8208133497b3778ede00ea463f',
500     'wiki.kaa.vec': 'd990d3b9bd511d2d630f923099a6b9110231b2ed',
501     'wiki.kab.vec': 'e3b73d41267d8d4cd42f6cc5a0c05dc4e021bf74',
502     'wiki.ka.vec': '8b92b73f27f9b77818211e053a33985589de7c62',
503     'wiki.kbd.vec': 'f5b8dbe47a7fae702232b5680b070ef6e865539e',
504     'wiki.kg.vec': '1550647b6059e6eb649b100e31c53bd0661117b2',
505     'wiki.ki.vec': 'c4e373e2ea13f7fa1e95b0733365e4b3fc8b2cc8',
506     'wiki.kj.vec': 'c27e563683f9c96ff6f680a6d6bb9e9e2f9960d0',
507     'wiki.kk.vec': '6343b2b31bad2e13d03a110b91c38fab4adc01cd',
508     'wiki.kl.vec': 'e5def7fb1b56c5956b6e951e912d53ba0ff089f8',
509     'wiki.km.vec': '64f7fff1df90b1f7241b232e901f76223a3719e0',
510     'wiki.kn.vec': '32763f4f860f0d081f3aabf3e7d17b7858e7d877',
511     'wiki.koi.vec': '4001f0617fe0fdd3b22116b304f497b7b16c6e4c',
512     'wiki.ko.vec': '042c85a788c2778cca538cf716b8a78f0d7fa823',
513     'wiki.krc.vec': '0c6ef043d51e5f337a309804f1db180fa0bb2cb8',
514     'wiki.kr.vec': '25d5b4d5911a819c48328c48fb346417d07d4070',
515     'wiki.ksh.vec': '4c3bb4f12073532b6fb7cc6c2be5e53319ef5b65',
516     'wiki.ks.vec': '5056a87c4ee2d8bf0792436fc6b2b61648014de9',
517     'wiki.ku.vec': '4d3a2401527dd9ba6be2b0cd31f6cd3edebadce9',
518     'wiki.kv.vec': '164dc44d701b9d606a45f0b0446076adc3858dca',
519     'wiki.kw.vec': 'f9eaa35a7e4f077f6de85c7801f74582f91b52c1',
520     'wiki.ky.vec': '13b0ae3f23822317a0243bd9182105c631c834b3',
521     'wiki.lad.vec': 'c510e520cde97050bf1cbeb36f2b90e6348ceed4',
522     'wiki.la.vec': '9ea6286a0581084533db8d6ee96e0b7d15166543',
523     'wiki.lbe.vec': '283619d93255571f14fd4545bb0577979171b990',
524     'wiki.lb.vec': 'b146f23628c84e64314a35a5b6cc65a33777e22d',
525     'wiki.lez.vec': '8e579b984a500ad89fc66767bfd7319766bd669b',
526     'wiki.lg.vec': 'b096f5248dfbb343dc4696c97ea253510e1c4ef9',
527     'wiki.lij.vec': '4ff5bb405c820e4119f0636efc301da15a08c00a',
528     'wiki.li.vec': '0fb9ec4ac93676d8ef651692062bc3d7f6ae0843',
529     'wiki.lmo.vec': 'a89414d9ceee4823622258f18936f67faf7e06e7',
530     'wiki.ln.vec': '70b6a286b42958e25cb80824e0d8f1aee2de6dde',
531     'wiki.lo.vec': '7c83f82b80c49b8eab21f62ecdb3681b8bda40a6',
532     'wiki.lrc.vec': 'c1ae4fb79a19d44bfe8f601f0a30fbec841fa612',
533     'wiki.ltg.vec': 'ec2f13d1290bd54afcaa74569e66e43e9bfef264',
534     'wiki.lt.vec': '58d3ebef24e5e31be1a8318b45c08ebb16ad775a',
535     'wiki.lv.vec': 'ef6b549f96e22718f513d47a611d3d6bc001a164',
536     'wiki.mai.vec': '7f513ff36e485b19f91f83b30c32dd82e9e497f6',
537     'wiki.map_bms.vec': 'e7deab5fdd38fa3331b1bcb4a16432b38c512e21',
538     'wiki.mdf.vec': 'b16099ce0283a241339716eac41cfd99fdea7f36',
539     'wiki.mg.vec': '0808252740909d6129f672584311263e7b2adadc',
540     'wiki.mhr.vec': '39f62e292336cabc364f0d1913540b881b406393',
541     'wiki.mh.vec': '7d2d8bff722fe0a5d869d9da11792a406aff3dc3',
542     'wiki.min.vec': '3bb0fa596cf27a1d165c55684bebdc8d40cb8ad7',
543     'wiki.mi.vec': 'e8acf9c7c2ab840a192c563aa776201a88e4ca89',
544     'wiki.mk.vec': '85a3d3f13fa88ffde023d2326c65bdded4983dff',
545     'wiki.ml.vec': '2b70fe76e8cf199a18551de782784a21e8db0b66',
546     'wiki.mn.vec': '7cef7ecdf9d98484d9b598b25d0e717dba6acfd9',
547     'wiki.mo.vec': 'cc54b661aefabdf516b49d24acb51273b3acf210',
548     'wiki.mrj.vec': 'aa1c1ecba1ffd6b42c8d9659a8a04ab328ae1650',
549     'wiki.mr.vec': '2cd6cf88bfdfb24850d345749ce0cfea8d65829e',
550     'wiki.ms.vec': '458e1a079799a54cdc0a7b78c7fa1729d2683a6d',
551     'wiki.mt.vec': '81f4c1d84dd4cc4276d59cb903fcc9aba46be981',
552     'wiki.multi.ar.vec': 'f1f12cc9d629382af574a3db74fe49c2fd615c8f',
553     'wiki.multi.bg.vec': '22470e664e4b35761a33c64433ea2f0c12140673',
554     'wiki.multi.ca.vec': 'bc8d98b4d86d740d1985d73d211d887d561bcdd7',
555     'wiki.multi.cs.vec': '17358b62e63f96b0479d6a70e9235a0421493884',
556     'wiki.multi.da.vec': 'ebc75f428714d26fb1fa31accce49ad3b31e273b',
557     'wiki.multi.de.vec': 'b9a63406aedf4446b467b94d12674bfe4723b52d',
558     'wiki.multi.el.vec': '03d33db85bf83f35b943ce93b18c02fa98a0bc05',
559     'wiki.multi.en.vec': '696719afdbe470ee4a2eb668229486dba1df19cc',
560     'wiki.multi.es.vec': '98c9e35564ec57fee5dbc6155890150452f45d3f',
561     'wiki.multi.et.vec': 'db10189093387e853f2fd3978770e1cc7bc07820',
562     'wiki.multi.fi.vec': '746916885a1c7d4ec3f139a32cf267f9e15f5363',
563     'wiki.multi.fr.vec': 'fe1535827b631d934beb02f8d36ba901b2c94a46',
564     'wiki.multi.he.vec': '6dd112f018165317da22971a2b6fdb2a15dafa91',
565     'wiki.multi.hr.vec': 'ff9f23cf595ec8dd93cd93c6b48049730c34253b',
566     'wiki.multi.hu.vec': '6da405c9b048f3cbb990bfb29ef149f0430aa2e7',
567     'wiki.multi.id.vec': '34edadab182682198c37ade8538530c545635742',
568     'wiki.multi.it.vec': 'c55802bd73d46a6fc86771097670e02a70b5d46d',
569     'wiki.multi.mk.vec': 'cec8550503ebca0bdc7ad11f2c15085b7072a990',
570     'wiki.multi.nl.vec': 'c3f45a5fe8a8bc213cdf35dce51651b752ca60c4',
571     'wiki.multi.no.vec': '105236df530c8fc2ce5b1e2550a2059bbc46fc28',
572     'wiki.multi.pl.vec': '676eb5acb22982c0c9a7d6e4c90d26730c6d120e',
573     'wiki.multi.pt.vec': '625b0a5384873c79a5dcfff5ee3fde49a3a65013',
574     'wiki.multi.ro.vec': '82bd59674509b69f988f9870e3a291836ba43e84',
575     'wiki.multi.ru.vec': 'a7d9c5f2ab2abb448a5111d352caa921adabe830',
576     'wiki.multi.sk.vec': '98d849ee77f0320472cc5afa002bfde129be7089',
577     'wiki.multi.sl.vec': 'fb5cfb8a9c44380d74fb21ddd204e820c4e05c31',
578     'wiki.multi.sv.vec': '95d6cc3ba23dffff9be6adb467b617dd57780cb2',
579     'wiki.multi.tr.vec': 'ecb0e353eaccba3fcacc6994d93065934ef429e9',
580     'wiki.multi.uk.vec': '35f4f5a1ead8bd66bcaf865021fc3aae94456ab6',
581     'wiki.multi.vi.vec': 'b1abe06360e1d65a0db65dd41ead7b2f9d651ea0',
582     'wiki.mus.vec': 'fa1066f7bd09df4589993ca498c19aeb6cf986fd',
583     'wiki.mwl.vec': '3d10a218242b94fcc3981aa3beb012b701827a55',
584     'wiki.my.vec': 'e7c7989e32b23ca1a9caf534cc65ecaf9e1b9112',
585     'wiki.myv.vec': '7de0927fd3d65677de7f770b3bd57c73b58df85d',
586     'wiki.mzn.vec': 'aefad49237808acab99e1ca8eeaaf531666f261d',
587     'wiki.nah.vec': 'c52e01cf4479fb7ec91ef39f298e8f97aeb6496e',
588     'wiki.nap.vec': '6c9bd8ce1e85ee679b25189fd6f6d36afb119b6c',
589     'wiki.na.vec': '8a592eb3dbe5693372714dff495d01cabc3ea215',
590     'wiki.nds_nl.vec': '1cd96d12e78e5cd3f65ca2773a17696bda387b9f',
591     'wiki.nds.vec': '7bf293149c08226e05bcf0442ac6e601162b9ffd',
592     'wiki.ne.vec': '1045d7876f947cd4602d9ca79f7c4323a5d3a52d',
593     'wiki-news-300d-1M-subword.vec': '717a3058e0ba5ef3cde52c3df0d4f0f60b0a113a',
594     'wiki-news-300d-1M.vec': '11cac9efe6f599e659be182f5766d6fbd5b1cab9',
595     'wiki.new.vec': '51f6c0b4ef1aee9fad4ab1cb69a7479db35e39a5',
596     'wiki.ng.vec': 'c3016cc07d40bd43bea84b7c600244ff3d2a928e',
597     'wiki.nl.vec': 'd796ee27e37b7d1d464e03c265c31ab62b52533e',
598     'wiki.nn.vec': '35aeab89ffeca0377accbbd3bf18b81913c75448',
599     'wiki.no.vec': 'd52e8019d7cc48569c8c3b514d2b1bd10261b5c0',
600     'wiki.nov.vec': '5455c6e8463b1c43dd073e3e177702fb9a1dd834',
601     'wiki.nrm.vec': 'b4cb941b126b26fa045c5fc75a490a31a969101c',
602     'wiki.nso.vec': 'a906271509c2b343df35d1471509492bbfa883aa',
603     'wiki.nv.vec': 'f5a6ea213bfe95c82cb22b53b4965df8b67ffeab',
604     'wiki.ny.vec': '3aec3dcaea6c35f8254c407621644f87df37e411',
605     'wiki.oc.vec': 'cc1833492899d75571148c2c305591f53d63f0b1',
606     'wiki.olo.vec': 'cbadb4cada4dc579d0becdac93dfb479d76bf6c8',
607     'wiki.om.vec': '91789a8d9f9284f7e71e4bb8d9a60eae4af4adca',
608     'wiki.or.vec': 'a6b120fe536b6c0133b077dca0043c3bc97eef0b',
609     'wiki.os.vec': '791b26cc300e9a1f0a08c7b2213a264e41ce30d6',
610     'wiki.pag.vec': '03f71faf060c4eb33802275279967349c0337553',
611     'wiki.pam.vec': '8fbd31e70d0ca0c61eb1a152efaa8ecb29180967',
612     'wiki.pap.vec': '8cd98267cc55a4f9de80212e29651ddf7a9e83fd',
613     'wiki.pa.vec': '4939d0db77a5b28d7d5aab0fab4f999d93b2053e',
614     'wiki.pcd.vec': 'd2e8e7321b6f1bce94c563cb8ef8af2b45cc3e48',
615     'wiki.pdc.vec': '401e24d0fb9b0ae9e06a5c700684361f58727fcf',
616     'wiki.pfl.vec': '0ad9b7f3ae13f909f12835107432fee4c4ed3031',
617     'wiki.pih.vec': '4ae6ef2a9c6c88e9322eda900e0f58be5592a29b',
618     'wiki.pi.vec': 'd388db284357042f4260e1a567cb489b05bb8e0b',
619     'wiki.pl.vec': 'd031adb6f83eda0364a861dcbf5ef779b5951c0b',
620     'wiki.pms.vec': 'e30bda8d33d61db43243c157b9ac2feeaff316c8',
621     'wiki.pnb.vec': '35f38862d3d83012d6db7baa8a4105e3e0a416e7',
622     'wiki.pnt.vec': '38134772012d68f247e34daf220d9d4ed3e7f489',
623     'wiki.ps.vec': '64f1bec5d5b937289199ceae2e1da6557ce48852',
624     'wiki.pt.vec': '7f11ebdb0cbf5929b38319f1e977d2c13bcd741b',
625     'wiki.qu.vec': '58de8c8290e8bc8f2a6a677312e28457113437b2',
626     'wiki.rm.vec': '5d3144b47a0dd98648a6df0636384ab2a010ad7b',
627     'wiki.rmy.vec': '3d36d3485961900c23355a0f7c2ba656a8558c29',
628     'wiki.rn.vec': '80b6171b78dd932f59f70dbef074abb906af4eee',
629     'wiki.roa_rup.vec': 'e31a44353cd84b976586c8df35a2ab58318120f0',
630     'wiki.roa_tara.vec': 'b3fcb01ff0bac53a0ba08c5c0c411f26ee83a95a',
631     'wiki.ro.vec': 'c088ea2752d5ec8b42e32410c191a14839ae8a1f',
632     'wiki.rue.vec': 'fe539e0ea0bbbfd3ee06bd0c5521a035c7361ec5',
633     'wiki.ru.vec': '7514a2c60ee4118abb451ed32a0d61cb52dec384',
634     'wiki.rw.vec': 'af2ec410da6519a86ba21004c8b4c7fde768a91c',
635     'wiki.sah.vec': '202470467194a1cbdcd571b14ef68371a29b38d9',
636     'wiki.sa.vec': '7fed78d1d7674453b9876ee99aeeeba85ea46699',
637     'wiki.scn.vec': 'bde043a235551e1643506774c5d9b61ecf2fc424',
638     'wiki.sco.vec': '4625a5ad90a57f994be9b3aa4f8f3ecda941a821',
639     'wiki.sc.vec': 'dba8dc7754ef04b1ba0cd702d94eea9575cde91c',
640     'wiki.sd.vec': '36852d1253496e598fbd9b9009f07f454a6bea5b',
641     'wiki.se.vec': 'f46b35ee6b893c2f12dd1b929bbc2b8120cbcd8d',
642     'wiki.sg.vec': '90ece136bef7ad6e4e97776a1c7238499544405d',
643     'wiki.sh.vec': '016691ecb26ace442731d92b1265e5c6c3d8ca5f',
644     'wiki.simple.vec': '55267c50fbdf4e4ae0fbbda5c73830a379d68795',
645     'wiki.si.vec': 'd05ed6a0bc1ee56e5d2e5f881d47372095f6eb0c',
646     'wiki.sk.vec': '98759aacf7352d49a51390fae02030776510ae13',
647     'wiki.sl.vec': 'b26997c0ed1de26a47b11efdc26ac1e7f189fa54',
648     'wiki.sm.vec': '88c2c57ca483626b052403418cb4372d72352bc9',
649     'wiki.sn.vec': '8dbb1019dcc8f842a8c0f550295ae697f8e1b7e0',
650     'wiki.so.vec': '294756b60b03fe57cb08abd8d677d6a717b40bc8',
651     'wiki.sq.vec': 'd07ffed553f5eb4756d0a1548a7ba9a51a52f7c6',
652     'wiki.srn.vec': 'faee05e550f5b08809a9ae5586ac4b08c9a1c359',
653     'wiki.sr.vec': '3cf09f476f55a92fdd2880f7ba336656ab232736',
654     'wiki.ss.vec': '488546a3b2f88f549c50ae9f32f1997cc441b039',
655     'wiki.stq.vec': '1bf88af29f1d86cac16042a5bea6b1651c96a8c1',
656     'wiki.st.vec': '963646055d12873b1c83b0eef8649ecaf473d42e',
657     'wiki.su.vec': '25e864495acb6d280bab0e62480f68550c9ceed4',
658     'wiki.sv.vec': 'eab83ae36701139696477b91b6e8d292ef175053',
659     'wiki.sw.vec': '8e70d207dbbd14e60a48e260a23fbf284a8e9f06',
660     'wiki.szl.vec': '0573cf888ec70b459b0596d34814fe60fd69f190',
661     'wiki.ta.vec': 'b66b5358527b1f3a6a421ab26464a3c1e75e18af',
662     'wiki.tcy.vec': '388b1d89642fcc790b688e9643b3d19e14d66f40',
663     'wiki.tet.vec': 'f38fe0e76b9b08ff652689eeee42c4fdadd9a47e',
664     'wiki.te.vec': 'e71dcf3cc45da1bcdae5e431324025bd2026d0c8',
665     'wiki.tg.vec': '6a5cd5bfe571ca0359b66d21bf6950553213f42d',
666     'wiki.th.vec': '1d6e0d525392a1042d017534f6c320c5a0afd345',
667     'wiki.ti.vec': 'c769fbc99bbb4138a40231e573685c7948d4a4c4',
668     'wiki.tk.vec': '33ae577f77d339ab7a0dff88855b8d5c974d0aef',
669     'wiki.tl.vec': 'd508e229ced7201510999e76d583de3ff2339d8b',
670     'wiki.tn.vec': '39f45f3fa86645bb25c54150204abcd51cc1048c',
671     'wiki.to.vec': '64d512665b55e9ef9a3915e8167347be79310fa0',
672     'wiki.tpi.vec': '407b96d235f54f3e0be9dc23a3bab89c6593a621',
673     'wiki.tr.vec': '13234aa1bf5f99e81d933482b3b83c3e4bf6c85e',
674     'wiki.ts.vec': '00f8229e2f230afd388221c0f823a1de9fc0e443',
675     'wiki.tt.vec': '913bb3a11da6f8142b3bbec3ef065162d9350f1d',
676     'wiki.tum.vec': 'bfbe43364724af882a520d2edcc2ce049c7357cd',
677     'wiki.tw.vec': 'f329b667d70d9f0b753e55e1b1579b5a5191d3bd',
678     'wiki.ty.vec': 'b881f60b8c75a71864d9847a17961d368f3058fc',
679     'wiki.tyv.vec': 'e8f9a36dc58e4108c553f96e247a877a099ab5ba',
680     'wiki.udm.vec': '336a8526f22e177faac69573661dc9c3ce36591f',
681     'wiki.ug.vec': '586d2febafaf17c9187c599ffd7b96e559103c34',
682     'wiki.uk.vec': '77f7737b9f88eac2b3e130ea8abb8886336fd0c6',
683     'wiki.ur.vec': 'cb8132102152a958df72bd3e25f1a72abb4c9c76',
684     'wiki.uz.vec': '11c3a76dae12b454f693811e33ae2e60015743e2',
685     'wiki.vec.vec': 'ae4b055fba21974e56beecab3a95f9dc24a62fd0',
686     'wiki.vep.vec': 'a38a781fde24f4d7b52aa8bc450b9949dd4e1808',
687     'wiki.ve.vec': 'b7d2947501de1c30a9f8496d5efae20c051104e1',
688     'wiki.vi.vec': 'bc84245b52b2e212e28dc6856c0693ce9845a9c5',
689     'wiki.vls.vec': '07e8636908c057b9870ce4b98c7130d460cf882a',
690     'wiki.vo.vec': 'c830988b6965bfce2f932b1be193f7d1f755f411',
691     'wiki.war.vec': '1f5d443d6f612b59a53820dd6f39fd886a6ad30f',
692     'wiki.wa.vec': '18f9ca1a585e1d18c3630029141a2e19d7d34a8e',
693     'wiki.wo.vec': '2ad96a7a9e640bc0dbcf316b1f414b92802dcb8e',
694     'wiki.wuu.vec': 'e1cbae1d3ad52329d0f36ada764016fbacf07049',
695     'wiki.xal.vec': 'b738222d84cb8c8fdb2b30a7219aa5d3bdc2f61c',
696     'wiki.xh.vec': 'bf37f741b0b75953281d11df2b4d80100df9e666',
697     'wiki.xmf.vec': 'dc1923cfd1a7002d5d60426b60e6756854ab4a14',
698     'wiki.yi.vec': '299d61958b7dcc38774768f1489121384726d860',
699     'wiki.yo.vec': 'e35c8aff2924ba07936be9d0d94bd298f09702a4',
700     'wiki.za.vec': 'e3a0e58bd2e5b1891c71f1f7e37ff71997a20361',
701     'wiki.zea.vec': 'ee12db26aab3f2b3b2745a298ef414e7aeb5a058',
702     'wiki.zh_classical.vec': '840981c83dd8e5cb02d1cd695e2fe0870941316c',
703     'wiki.zh_min_nan.vec': 'f91ccb013e200bb7ed560082ddf4bdd9c2f315bb',
704     'wiki.zh.vec': '117ab34faa80e381641fbabf3a24bc8cfba44050',
705     'wiki.zh_yue.vec': 'd2ac1ab9eb1a908797644f83f259c90cb3c1a350',
706     'wiki.zu.vec': '4b244b9697a8280e6646842c5fc81bb3a6bc8ec7'}
707
Full Screen

compress.py

Source: compress.py Github

copy
1#!/usr/bin/python
2# Copyright (C) 2012 Ion Torrent Systems, Inc. All Rights Reserved
3
4from ion.utils.blockprocessing import printtime
5import os
6import traceback
7
8import zipfile
9
10
11def make_zip(zip_file, to_zip, arcname=None, use_sys_zip=True, compressed=True):
12    """Try to make a zip of a file if it exists
13    zip_file is the name of the archive file,
14    to_zip is the name of the file to add to the archive,
15    arcname is optional; renames the file in the archive,
16    use_sys_zip flag will call 'zip' shell command to create archive"""
17    # bug in python 2.6 with large zip files; use system zip until its fixed.
18    #    printtime("Start make_zip on %s" % to_zip)
19
20    try:
21        compression = zipfile.ZIP_DEFLATED
22    except Exception:
23        compression = zipfile.ZIP_STORED
24
25    if not compressed:
26        compression = zipfile.ZIP_STORED
27        printtime("not compressed")
28    else:
29        printtime("compressed")
30
31    if os.path.exists(to_zip):
32        if use_sys_zip:
33            if arcname != None:
34                # This is a hack to trigger only storing of names, no directory heirarchy
35                cmd = "zip --junk-paths %s %s" % (zip_file, to_zip)
36            else:
37                cmd = "zip %s %s" % (zip_file, to_zip)
38            try:
39                os.system(cmd)
40            except Exception:
41                printtime("Error executing:\n%s" % cmd)
42                print(traceback.format_exc())
43        else:
44            zf = zipfile.ZipFile(zip_file, mode="a", allowZip64=True)
45            try:
46                # adding file with compression
47                if arcname == None:
48                    zf.write(to_zip, compress_type=compression)
49                else:
50                    zf.write(to_zip, arcname, compress_type=compression)
51            #                print "Created ", zip_file, " of", to_zip
52            except OSError:
53                print("OSError with - :", to_zip)
54            except zipfile.LargeZipFile:
55                printtime(
56                    "The zip file was too large, ZIP64 extensions could not be enabled"
57                )
58            except:
59                printtime("Unexpected error creating zip")
60                traceback.print_exc()
61            finally:
62                zf.close()
63    #        printtime("End make_zip %s" % to_zip)
64    else:
65        printtime(
66            "File %s not found.  Zipfile %s not created/updated."
67            % (str(to_zip), str(zip_file))
68        )
69
Full Screen

script_helper.py

Source: script_helper.py Github

copy
1# Common utility functions used by various script execution tests
2#  e.g. test_cmd_line, test_cmd_line_script and test_runpy
3
4import sys
5import os
6import re
7import os.path
8import tempfile
9import subprocess
10import py_compile
11import contextlib
12import shutil
13import zipfile
14
15from imp import source_from_cache
16from test.support import make_legacy_pyc, strip_python_stderr
17
18# Executing the interpreter in a subprocess
19def _assert_python(expected_success, *args, **env_vars):
20    cmd_line = [sys.executable]
21    if not env_vars:
22        cmd_line.append('-E')
23    cmd_line.extend(args)
24    # Need to preserve the original environment, for in-place testing of
25    # shared library builds.
26    env = os.environ.copy()
27    env.update(env_vars)
28    p = subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
29                         stdout=subprocess.PIPE, stderr=subprocess.PIPE,
30                         env=env)
31    try:
32        out, err = p.communicate()
33    finally:
34        subprocess._cleanup()
35        p.stdout.close()
36        p.stderr.close()
37    rc = p.returncode
38    err =  strip_python_stderr(err)
39    if (rc and expected_success) or (not rc and not expected_success):
40        raise AssertionError(
41            "Process return code is %d, "
42            "stderr follows:\n%s" % (rc, err.decode('ascii', 'ignore')))
43    return rc, out, err
44
45def assert_python_ok(*args, **env_vars):
46    """
47    Assert that running the interpreter with `args` and optional environment
48    variables `env_vars` is ok and return a (return code, stdout, stderr) tuple.
49    """
50    return _assert_python(True, *args, **env_vars)
51
52def assert_python_failure(*args, **env_vars):
53    """
54    Assert that running the interpreter with `args` and optional environment
55    variables `env_vars` fails and return a (return code, stdout, stderr) tuple.
56    """
57    return _assert_python(False, *args, **env_vars)
58
59def spawn_python(*args):
60    cmd_line = [sys.executable, '-E']
61    cmd_line.extend(args)
62    return subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
63                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
64
65def kill_python(p):
66    p.stdin.close()
67    data = p.stdout.read()
68    p.stdout.close()
69    # try to cleanup the child so we don't appear to leak when running
70    # with regrtest -R.
71    p.wait()
72    subprocess._cleanup()
73    return data
74
75# Script creation utilities
76@contextlib.contextmanager
77def temp_dir():
78    dirname = tempfile.mkdtemp()
79    dirname = os.path.realpath(dirname)
80    try:
81        yield dirname
82    finally:
83        shutil.rmtree(dirname)
84
85def make_script(script_dir, script_basename, source):
86    script_filename = script_basename+os.extsep+'py'
87    script_name = os.path.join(script_dir, script_filename)
88    # The script should be encoded to UTF-8, the default string encoding
89    script_file = open(script_name, 'w', encoding='utf-8')
90    script_file.write(source)
91    script_file.close()
92    return script_name
93
94def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None):
95    zip_filename = zip_basename+os.extsep+'zip'
96    zip_name = os.path.join(zip_dir, zip_filename)
97    zip_file = zipfile.ZipFile(zip_name, 'w')
98    if name_in_zip is None:
99        parts = script_name.split(os.sep)
100        if len(parts) >= 2 and parts[-2] == '__pycache__':
101            legacy_pyc = make_legacy_pyc(source_from_cache(script_name))
102            name_in_zip = os.path.basename(legacy_pyc)
103            script_name = legacy_pyc
104        else:
105            name_in_zip = os.path.basename(script_name)
106    zip_file.write(script_name, name_in_zip)
107    zip_file.close()
108    #if test.support.verbose:
109    #    zip_file = zipfile.ZipFile(zip_name, 'r')
110    #    print 'Contents of %r:' % zip_name
111    #    zip_file.printdir()
112    #    zip_file.close()
113    return zip_name, os.path.join(zip_name, name_in_zip)
114
115def make_pkg(pkg_dir, init_source=''):
116    os.mkdir(pkg_dir)
117    make_script(pkg_dir, '__init__', init_source)
118
119def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
120                 source, depth=1, compiled=False):
121    unlink = []
122    init_name = make_script(zip_dir, '__init__', '')
123    unlink.append(init_name)
124    init_basename = os.path.basename(init_name)
125    script_name = make_script(zip_dir, script_basename, source)
126    unlink.append(script_name)
127    if compiled:
128        init_name = py_compile(init_name, doraise=True)
129        script_name = py_compile(script_name, doraise=True)
130        unlink.extend((init_name, script_name))
131    pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)]
132    script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name))
133    zip_filename = zip_basename+os.extsep+'zip'
134    zip_name = os.path.join(zip_dir, zip_filename)
135    zip_file = zipfile.ZipFile(zip_name, 'w')
136    for name in pkg_names:
137        init_name_in_zip = os.path.join(name, init_basename)
138        zip_file.write(init_name, init_name_in_zip)
139    zip_file.write(script_name, script_name_in_zip)
140    zip_file.close()
141    for name in unlink:
142        os.unlink(name)
143    #if test.support.verbose:
144    #    zip_file = zipfile.ZipFile(zip_name, 'r')
145    #    print 'Contents of %r:' % zip_name
146    #    zip_file.printdir()
147    #    zip_file.close()
148    return zip_name, os.path.join(zip_name, script_name_in_zip)
149
Full Screen

SEA3DDeflate.js

Source: SEA3DDeflate.js Github

copy
1/*
2 * $Id: rawinflate.js,v 0.3 2013/04/09 14:25:38 dankogai Exp dankogai $
3 *
4 * GNU General Public License, version 2 (GPL-2.0)
5 *   http://opensource.org/licenses/GPL-2.0
6 * original:
7 *   http://www.onicos.com/staff/iz/amuse/javascript/expert/inflate.txt
8 */
9
10(function(ctx){
11
12/* Copyright (C) 1999 Masanao Izumo <[email protected]>
13 * Version: 1.0.0.1
14 * LastModified: Dec 25 1999
15 */
16
17/* Interface:
18 * data = zip_inflate(src);
19 */
20
21/* constant parameters */
22var zip_WSIZE = 32768;		// Sliding Window size
23var zip_STORED_BLOCK = 0;
24var zip_STATIC_TREES = 1;
25var zip_DYN_TREES    = 2;
26
27/* for inflate */
28var zip_lbits = 9; 		// bits in base literal/length lookup table
29var zip_dbits = 6; 		// bits in base distance lookup table
30var zip_INBUFSIZ = 32768;	// Input buffer size
31var zip_INBUF_EXTRA = 64;	// Extra buffer
32
33/* variables (inflate) */
34var zip_slide;
35var zip_wp;			// current position in slide
36var zip_fixed_tl = null;	// inflate static
37var zip_fixed_td;		// inflate static
38var zip_fixed_bl, fixed_bd;	// inflate static
39var zip_bit_buf;		// bit buffer
40var zip_bit_len;		// bits in bit buffer
41var zip_method;
42var zip_eof;
43var zip_copy_leng;
44var zip_copy_dist;
45var zip_tl, zip_td;	// literal/length and distance decoder tables
46var zip_bl, zip_bd;	// number of bits decoded by tl and td
47
48var zip_inflate_data;
49var zip_inflate_pos;
50
51
52/* constant tables (inflate) */
53var zip_MASK_BITS = new Array(
54    0x0000,
55    0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff,
56    0x01ff, 0x03ff, 0x07ff, 0x0fff, 0x1fff, 0x3fff, 0x7fff, 0xffff);
57// Tables for deflate from PKZIP's appnote.txt.
58var zip_cplens = new Array( // Copy lengths for literal codes 257..285
59    3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31,
60    35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0);
61/* note: see note #13 above about the 258 in this list. */
62var zip_cplext = new Array( // Extra bits for literal codes 257..285
63    0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2,
64    3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 99, 99); // 99==invalid
65var zip_cpdist = new Array( // Copy offsets for distance codes 0..29
66    1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193,
67    257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145,
68    8193, 12289, 16385, 24577);
69var zip_cpdext = new Array( // Extra bits for distance codes
70    0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6,
71    7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
72    12, 12, 13, 13);
73var zip_border = new Array(  // Order of the bit length code lengths
74    16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15);
75/* objects (inflate) */
76
77var zip_HuftList = function() {
78    this.next = null;
79    this.list = null;
80}
81
82var zip_HuftNode = function() {
83    this.e = 0; // number of extra bits or operation
84    this.b = 0; // number of bits in this code or subcode
85
86    // union
87    this.n = 0; // literal, length base, or distance base
88    this.t = null; // (zip_HuftNode) pointer to next level of table
89}
90
91var zip_HuftBuild = function(b,	// code lengths in bits (all assumed <= BMAX)
92		       n,	// number of codes (assumed <= N_MAX)
93		       s,	// number of simple-valued codes (0..s-1)
94		       d,	// list of base values for non-simple codes
95		       e,	// list of extra bits for non-simple codes
96		       mm	// maximum lookup bits
97		   ) {
98    this.BMAX = 16;   // maximum bit length of any code
99    this.N_MAX = 288; // maximum number of codes in any set
100    this.status = 0;	// 0: success, 1: incomplete table, 2: bad input
101    this.root = null;	// (zip_HuftList) starting table
102    this.m = 0;		// maximum lookup bits, returns actual
103
104/* Given a list of code lengths and a maximum table size, make a set of
105   tables to decode that set of codes.	Return zero on success, one if
106   the given code set is incomplete (the tables are still built in this
107   case), two if the input is invalid (all zero length codes or an
108   oversubscribed set of lengths), and three if not enough memory.
109   The code with value 256 is special, and the tables are constructed
110   so that no bits beyond that code are fetched when that code is
111   decoded. */
112    {
113	var a;			// counter for codes of length k
114	var c = new Array(this.BMAX+1);	// bit length count table
115	var el;			// length of EOB code (value 256)
116	var f;			// i repeats in table every f entries
117	var g;			// maximum code length
118	var h;			// table level
119	var i;			// counter, current code
120	var j;			// counter
121	var k;			// number of bits in current code
122	var lx = new Array(this.BMAX+1);	// stack of bits per table
123	var p;			// pointer into c[], b[], or v[]
124	var pidx;		// index of p
125	var q;			// (zip_HuftNode) points to current table
126	var r = new zip_HuftNode(); // table entry for structure assignment
127	var u = new Array(this.BMAX); // zip_HuftNode[BMAX][]  table stack
128	var v = new Array(this.N_MAX); // values in order of bit length
129	var w;
130	var x = new Array(this.BMAX+1);// bit offsets, then code stack
131	var xp;			// pointer into x or c
132	var y;			// number of dummy codes added
133	var z;			// number of entries in current table
134	var o;
135	var tail;		// (zip_HuftList)
136
137	tail = this.root = null;
138	for(i = 0; i < c.length; i++)
139	    c[i] = 0;
140	for(i = 0; i < lx.length; i++)
141	    lx[i] = 0;
142	for(i = 0; i < u.length; i++)
143	    u[i] = null;
144	for(i = 0; i < v.length; i++)
145	    v[i] = 0;
146	for(i = 0; i < x.length; i++)
147	    x[i] = 0;
148
149	// Generate counts for each bit length
150	el = n > 256 ? b[256] : this.BMAX; // set length of EOB code, if any
151	p = b; pidx = 0;
152	i = n;
153	do {
154	    c[p[pidx]]++;	// assume all entries <= BMAX
155	    pidx++;
156	} while(--i > 0);
157	if(c[0] == n) {	// null input--all zero length codes
158	    this.root = null;
159	    this.m = 0;
160	    this.status = 0;
161	    return;
162	}
163
164	// Find minimum and maximum length, bound *m by those
165	for(j = 1; j <= this.BMAX; j++)
166	    if(c[j] != 0)
167		break;
168	k = j;			// minimum code length
169	if(mm < j)
170	    mm = j;
171	for(i = this.BMAX; i != 0; i--)
172	    if(c[i] != 0)
173		break;
174	g = i;			// maximum code length
175	if(mm > i)
176	    mm = i;
177
178	// Adjust last length count to fill out codes, if needed
179	for(y = 1 << j; j < i; j++, y <<= 1)
180	    if((y -= c[j]) < 0) {
181		this.status = 2;	// bad input: more codes than bits
182		this.m = mm;
183		return;
184	    }
185	if((y -= c[i]) < 0) {
186	    this.status = 2;
187	    this.m = mm;
188	    return;
189	}
190	c[i] += y;
191
192	// Generate starting offsets into the value table for each length
193	x[1] = j = 0;
194	p = c;
195	pidx = 1;
196	xp = 2;
197	while(--i > 0)		// note that i == g from above
198	    x[xp++] = (j += p[pidx++]);
199
200	// Make a table of values in order of bit lengths
201	p = b; pidx = 0;
202	i = 0;
203	do {
204	    if((j = p[pidx++]) != 0)
205		v[x[j]++] = i;
206	} while(++i < n);
207	n = x[g];			// set n to length of v
208
209	// Generate the Huffman codes and for each, make the table entries
210	x[0] = i = 0;		// first Huffman code is zero
211	p = v; pidx = 0;		// grab values in bit order
212	h = -1;			// no tables yet--level -1
213	w = lx[0] = 0;		// no bits decoded yet
214	q = null;			// ditto
215	z = 0;			// ditto
216
217	// go through the bit lengths (k already is bits in shortest code)
218	for(; k <= g; k++) {
219	    a = c[k];
220	    while(a-- > 0) {
221		// here i is the Huffman code of length k bits for value p[pidx]
222		// make tables up to required level
223		while(k > w + lx[1 + h]) {
224		    w += lx[1 + h]; // add bits already decoded
225		    h++;
226
227		    // compute minimum size table less than or equal to *m bits
228		    z = (z = g - w) > mm ? mm : z; // upper limit
229		    if((f = 1 << (j = k - w)) > a + 1) { // try a k-w bit table
230			// too few codes for k-w bit table
231			f -= a + 1;	// deduct codes from patterns left
232			xp = k;
233			while(++j < z) { // try smaller tables up to z bits
234			    if((f <<= 1) <= c[++xp])
235				break;	// enough codes to use up j bits
236			    f -= c[xp];	// else deduct codes from patterns
237			}
238		    }
239		    if(w + j > el && w < el)
240			j = el - w;	// make EOB code end at table
241		    z = 1 << j;	// table entries for j-bit table
242		    lx[1 + h] = j; // set table size in stack
243
244		    // allocate and link in new table
245		    q = new Array(z);
246		    for(o = 0; o < z; o++) {
247			q[o] = new zip_HuftNode();
248		    }
249
250		    if(tail == null)
251			tail = this.root = new zip_HuftList();
252		    else
253			tail = tail.next = new zip_HuftList();
254		    tail.next = null;
255		    tail.list = q;
256		    u[h] = q;	// table starts after link
257
258		    /* connect to last table, if there is one */
259		    if(h > 0) {
260			x[h] = i;		// save pattern for backing up
261			r.b = lx[h];	// bits to dump before this table
262			r.e = 16 + j;	// bits in this table
263			r.t = q;		// pointer to this table
264			j = (i & ((1 << w) - 1)) >> (w - lx[h]);
265			u[h-1][j].e = r.e;
266			u[h-1][j].b = r.b;
267			u[h-1][j].n = r.n;
268			u[h-1][j].t = r.t;
269		    }
270		}
271
272		// set up table entry in r
273		r.b = k - w;
274		if(pidx >= n)
275		    r.e = 99;		// out of values--invalid code
276		else if(p[pidx] < s) {
277		    r.e = (p[pidx] < 256 ? 16 : 15); // 256 is end-of-block code
278		    r.n = p[pidx++];	// simple code is just the value
279		} else {
280		    r.e = e[p[pidx] - s];	// non-simple--look up in lists
281		    r.n = d[p[pidx++] - s];
282		}
283
284		// fill code-like entries with r //
285		f = 1 << (k - w);
286		for(j = i >> w; j < z; j += f) {
287		    q[j].e = r.e;
288		    q[j].b = r.b;
289		    q[j].n = r.n;
290		    q[j].t = r.t;
291		}
292
293		// backwards increment the k-bit code i
294		for(j = 1 << (k - 1); (i & j) != 0; j >>= 1)
295		    i ^= j;
296		i ^= j;
297
298		// backup over finished tables
299		while((i & ((1 << w) - 1)) != x[h]) {
300		    w -= lx[h];		// don't need to update q
301		    h--;
302		}
303	    }
304	}
305
306	/* return actual size of base table */
307	this.m = lx[1];
308
309	/* Return true (1) if we were given an incomplete table */
310	this.status = ((y != 0 && g != 1) ? 1 : 0);
311    } /* end of constructor */
312}
313
314
315/* routines (inflate) */
316
317var zip_GET_BYTE = function() {
318    if(zip_inflate_data.length == zip_inflate_pos)
319	return -1;
320	return zip_inflate_data[zip_inflate_pos++];
321}
322
323var zip_NEEDBITS = function(n) {
324    while(zip_bit_len < n) {
325	zip_bit_buf |= zip_GET_BYTE() << zip_bit_len;
326	zip_bit_len += 8;
327    }
328}
329
330var zip_GETBITS = function(n) {
331    return zip_bit_buf & zip_MASK_BITS[n];
332}
333
334var zip_DUMPBITS = function(n) {
335    zip_bit_buf >>= n;
336    zip_bit_len -= n;
337}
338
339var zip_inflate_codes = function(buff, off, size) {
340    /* inflate (decompress) the codes in a deflated (compressed) block.
341       Return an error code or zero if it all goes ok. */
342    var e;		// table entry flag/number of extra bits
343    var t;		// (zip_HuftNode) pointer to table entry
344    var n;
345
346    if(size == 0)
347      return 0;
348
349    // inflate the coded data
350    n = 0;
351    for(;;) {			// do until end of block
352	zip_NEEDBITS(zip_bl);
353	t = zip_tl.list[zip_GETBITS(zip_bl)];
354	e = t.e;
355	while(e > 16) {
356	    if(e == 99)
357		return -1;
358	    zip_DUMPBITS(t.b);
359	    e -= 16;
360	    zip_NEEDBITS(e);
361	    t = t.t[zip_GETBITS(e)];
362	    e = t.e;
363	}
364	zip_DUMPBITS(t.b);
365
366	if(e == 16) {		// then it's a literal
367	    zip_wp &= zip_WSIZE - 1;
368	    buff[off + n++] = zip_slide[zip_wp++] = t.n;
369	    if(n == size)
370		return size;
371	    continue;
372	}
373
374	// exit if end of block
375	if(e == 15)
376	    break;
377
378	// it's an EOB or a length
379
380	// get length of block to copy
381	zip_NEEDBITS(e);
382	zip_copy_leng = t.n + zip_GETBITS(e);
383	zip_DUMPBITS(e);
384
385	// decode distance of block to copy
386	zip_NEEDBITS(zip_bd);
387	t = zip_td.list[zip_GETBITS(zip_bd)];
388	e = t.e;
389
390	while(e > 16) {
391	    if(e == 99)
392		return -1;
393	    zip_DUMPBITS(t.b);
394	    e -= 16;
395	    zip_NEEDBITS(e);
396	    t = t.t[zip_GETBITS(e)];
397	    e = t.e;
398	}
399	zip_DUMPBITS(t.b);
400	zip_NEEDBITS(e);
401	zip_copy_dist = zip_wp - t.n - zip_GETBITS(e);
402	zip_DUMPBITS(e);
403
404	// do the copy
405	while(zip_copy_leng > 0 && n < size) {
406	    zip_copy_leng--;
407	    zip_copy_dist &= zip_WSIZE - 1;
408	    zip_wp &= zip_WSIZE - 1;
409	    buff[off + n++] = zip_slide[zip_wp++]
410		= zip_slide[zip_copy_dist++];
411	}
412
413	if(n == size)
414	    return size;
415    }
416
417    zip_method = -1; // done
418    return n;
419}
420
421var zip_inflate_stored = function(buff, off, size) {
422    /* "decompress" an inflated type 0 (stored) block. */
423    var n;
424
425    // go to byte boundary
426    n = zip_bit_len & 7;
427    zip_DUMPBITS(n);
428
429    // get the length and its complement
430    zip_NEEDBITS(16);
431    n = zip_GETBITS(16);
432    zip_DUMPBITS(16);
433    zip_NEEDBITS(16);
434    if(n != ((~zip_bit_buf) & 0xffff))
435	return -1;			// error in compressed data
436    zip_DUMPBITS(16);
437
438    // read and output the compressed data
439    zip_copy_leng = n;
440
441    n = 0;
442    while(zip_copy_leng > 0 && n < size) {
443	zip_copy_leng--;
444	zip_wp &= zip_WSIZE - 1;
445	zip_NEEDBITS(8);
446	buff[off + n++] = zip_slide[zip_wp++] =
447	    zip_GETBITS(8);
448	zip_DUMPBITS(8);
449    }
450
451    if(zip_copy_leng == 0)
452      zip_method = -1; // done
453    return n;
454}
455
456var zip_inflate_fixed = function(buff, off, size) {
457    /* decompress an inflated type 1 (fixed Huffman codes) block.  We should
458       either replace this with a custom decoder, or at least precompute the
459       Huffman tables. */
460
461    // if first time, set up tables for fixed blocks
462    if(zip_fixed_tl == null) {
463	var i;			// temporary variable
464	var l = new Array(288);	// length list for huft_build
465	var h;	// zip_HuftBuild
466
467	// literal table
468	for(i = 0; i < 144; i++)
469	    l[i] = 8;
470	for(; i < 256; i++)
471	    l[i] = 9;
472	for(; i < 280; i++)
473	    l[i] = 7;
474	for(; i < 288; i++)	// make a complete, but wrong code set
475	    l[i] = 8;
476	zip_fixed_bl = 7;
477
478	h = new zip_HuftBuild(l, 288, 257, zip_cplens, zip_cplext,
479			      zip_fixed_bl);
480	if(h.status != 0) {
481	    alert("HufBuild error: "+h.status);
482	    return -1;
483	}
484	zip_fixed_tl = h.root;
485	zip_fixed_bl = h.m;
486
487	// distance table
488	for(i = 0; i < 30; i++)	// make an incomplete code set
489	    l[i] = 5;
490	zip_fixed_bd = 5;
491
492	h = new zip_HuftBuild(l, 30, 0, zip_cpdist, zip_cpdext, zip_fixed_bd);
493	if(h.status > 1) {
494	    zip_fixed_tl = null;
495	    alert("HufBuild error: "+h.status);
496	    return -1;
497	}
498	zip_fixed_td = h.root;
499	zip_fixed_bd = h.m;
500    }
501
502    zip_tl = zip_fixed_tl;
503    zip_td = zip_fixed_td;
504    zip_bl = zip_fixed_bl;
505    zip_bd = zip_fixed_bd;
506    return zip_inflate_codes(buff, off, size);
507}
508
509var zip_inflate_dynamic = function(buff, off, size) {
510    // decompress an inflated type 2 (dynamic Huffman codes) block.
511    var i;		// temporary variables
512    var j;
513    var l;		// last length
514    var n;		// number of lengths to get
515    var t;		// (zip_HuftNode) literal/length code table
516    var nb;		// number of bit length codes
517    var nl;		// number of literal/length codes
518    var nd;		// number of distance codes
519    var ll = new Array(286+30); // literal/length and distance code lengths
520    var h;		// (zip_HuftBuild)
521
522    for(i = 0; i < ll.length; i++)
523	ll[i] = 0;
524
525    // read in table lengths
526    zip_NEEDBITS(5);
527    nl = 257 + zip_GETBITS(5);	// number of literal/length codes
528    zip_DUMPBITS(5);
529    zip_NEEDBITS(5);
530    nd = 1 + zip_GETBITS(5);	// number of distance codes
531    zip_DUMPBITS(5);
532    zip_NEEDBITS(4);
533    nb = 4 + zip_GETBITS(4);	// number of bit length codes
534    zip_DUMPBITS(4);
535    if(nl > 286 || nd > 30)
536      return -1;		// bad lengths
537
538    // read in bit-length-code lengths
539    for(j = 0; j < nb; j++)
540    {
541	zip_NEEDBITS(3);
542	ll[zip_border[j]] = zip_GETBITS(3);
543	zip_DUMPBITS(3);
544    }
545    for(; j < 19; j++)
546	ll[zip_border[j]] = 0;
547
548    // build decoding table for trees--single level, 7 bit lookup
549    zip_bl = 7;
550    h = new zip_HuftBuild(ll, 19, 19, null, null, zip_bl);
551    if(h.status != 0)
552	return -1;	// incomplete code set
553
554    zip_tl = h.root;
555    zip_bl = h.m;
556
557    // read in literal and distance code lengths
558    n = nl + nd;
559    i = l = 0;
560    while(i < n) {
561	zip_NEEDBITS(zip_bl);
562	t = zip_tl.list[zip_GETBITS(zip_bl)];
563	j = t.b;
564	zip_DUMPBITS(j);
565	j = t.n;
566	if(j < 16)		// length of code in bits (0..15)
567	    ll[i++] = l = j;	// save last length in l
568	else if(j == 16) {	// repeat last length 3 to 6 times
569	    zip_NEEDBITS(2);
570	    j = 3 + zip_GETBITS(2);
571	    zip_DUMPBITS(2);
572	    if(i + j > n)
573		return -1;
574	    while(j-- > 0)
575		ll[i++] = l;
576	} else if(j == 17) {	// 3 to 10 zero length codes
577	    zip_NEEDBITS(3);
578	    j = 3 + zip_GETBITS(3);
579	    zip_DUMPBITS(3);
580	    if(i + j > n)
581		return -1;
582	    while(j-- > 0)
583		ll[i++] = 0;
584	    l = 0;
585	} else {		// j == 18: 11 to 138 zero length codes
586	    zip_NEEDBITS(7);
587	    j = 11 + zip_GETBITS(7);
588	    zip_DUMPBITS(7);
589	    if(i + j > n)
590		return -1;
591	    while(j-- > 0)
592		ll[i++] = 0;
593	    l = 0;
594	}
595    }
596
597    // build the decoding tables for literal/length and distance codes
598    zip_bl = zip_lbits;
599    h = new zip_HuftBuild(ll, nl, 257, zip_cplens, zip_cplext, zip_bl);
600    if(zip_bl == 0)	// no literals or lengths
601	h.status = 1;
602    if(h.status != 0) {
603	/*if(h.status == 1)
604	    ;// **incomplete literal tree** */
605	return -1;		// incomplete code set
606    }
607    zip_tl = h.root;
608    zip_bl = h.m;
609
610    for(i = 0; i < nd; i++)
611	ll[i] = ll[i + nl];
612    zip_bd = zip_dbits;
613    h = new zip_HuftBuild(ll, nd, 0, zip_cpdist, zip_cpdext, zip_bd);
614    zip_td = h.root;
615    zip_bd = h.m;
616
617    if(zip_bd == 0 && nl > 257) {   // lengths but no distances
618	// **incomplete distance tree**
619	return -1;
620    }
621
622    /*if(h.status == 1) {
623	;// **incomplete distance tree**
624    }*/
625    if(h.status != 0)
626	return -1;
627
628    // decompress until an end-of-block code
629    return zip_inflate_codes(buff, off, size);
630}
631
632var zip_inflate_start = function() {
633    var i;
634
635    if(zip_slide == null)
636	zip_slide = new Array(2 * zip_WSIZE);
637    zip_wp = 0;
638    zip_bit_buf = 0;
639    zip_bit_len = 0;
640    zip_method = -1;
641    zip_eof = false;
642    zip_copy_leng = zip_copy_dist = 0;
643    zip_tl = null;
644}
645
646var zip_inflate_internal = function(buff, off, size) {
647    // decompress an inflated entry
648    var n, i;
649
650    n = 0;
651    while(n < size) {
652	if(zip_eof && zip_method == -1)
653	    return n;
654
655	if(zip_copy_leng > 0) {
656	    if(zip_method != zip_STORED_BLOCK) {
657		// STATIC_TREES or DYN_TREES
658		while(zip_copy_leng > 0 && n < size) {
659		    zip_copy_leng--;
660		    zip_copy_dist &= zip_WSIZE - 1;
661		    zip_wp &= zip_WSIZE - 1;
662		    buff[off + n++] = zip_slide[zip_wp++] =
663			zip_slide[zip_copy_dist++];
664		}
665	    } else {
666		while(zip_copy_leng > 0 && n < size) {
667		    zip_copy_leng--;
668		    zip_wp &= zip_WSIZE - 1;
669		    zip_NEEDBITS(8);
670		    buff[off + n++] = zip_slide[zip_wp++] = zip_GETBITS(8);
671		    zip_DUMPBITS(8);
672		}
673		if(zip_copy_leng == 0)
674		    zip_method = -1; // done
675	    }
676	    if(n == size)
677		return n;
678	}
679
680	if(zip_method == -1) {
681	    if(zip_eof)
682		break;
683
684	    // read in last block bit
685	    zip_NEEDBITS(1);
686	    if(zip_GETBITS(1) != 0)
687		zip_eof = true;
688	    zip_DUMPBITS(1);
689
690	    // read in block type
691	    zip_NEEDBITS(2);
692	    zip_method = zip_GETBITS(2);
693	    zip_DUMPBITS(2);
694	    zip_tl = null;
695	    zip_copy_leng = 0;
696	}
697
698	switch(zip_method) {
699	  case 0: // zip_STORED_BLOCK
700	    i = zip_inflate_stored(buff, off + n, size - n);
701	    break;
702
703	  case 1: // zip_STATIC_TREES
704	    if(zip_tl != null)
705		i = zip_inflate_codes(buff, off + n, size - n);
706	    else
707		i = zip_inflate_fixed(buff, off + n, size - n);
708	    break;
709
710	  case 2: // zip_DYN_TREES
711	    if(zip_tl != null)
712		i = zip_inflate_codes(buff, off + n, size - n);
713	    else
714		i = zip_inflate_dynamic(buff, off + n, size - n);
715	    break;
716
717	  default: // error
718	    i = -1;
719	    break;
720	}
721
722	if(i == -1) {
723	    if(zip_eof)
724		return 0;
725	    return -1;
726	}
727	n += i;
728    }
729    return n;
730}
731
732var zip_inflate = function(data) {
733    var i, j, pos = 0;
734
735    zip_inflate_start();
736    zip_inflate_data = new Uint8Array(data);
737    zip_inflate_pos = 0;
738	
739    var buff = new Uint8Array(1024);
740	
741    var out = [];
742    while((i = zip_inflate_internal(buff, 0, buff.length)) > 0)
743		for(j = 0; j < i; j++)
744			out[pos++] = buff[j];		    
745	
746    zip_inflate_data = null; // G.C.
747    return new Uint8Array(out).buffer;
748}
749
750if (! ctx.RawDeflate) ctx.RawDeflate = {};
751ctx.RawDeflate.inflate = zip_inflate;
752
753})(this);
754
755/**
756 * 	SEA3D DEFLATE
757 * 	@author Sunag / http://www.sunag.com.br/
758 */
759
760SEA3D.File.DeflateUncompress = function( data ) {
761
762	return RawDeflate.inflate( data );
763
764}
765
766SEA3D.File.setDecompressionEngine( 1, "deflate", SEA3D.File.DeflateUncompress );
Full Screen

users.js

Source: users.js Github

copy
1const mongoose = require('mongoose');
2const User = mongoose.model('User');
3const bcrypt = require('bcrypt');
4const session = require('express-session');
5
6mongoose.Promise = global.Promise;
7
8module.exports = {
9    strZip: function(req, res){
10        User.find({ _id: req.params.id }, function(err, user) {
11            if(err){
12                res.json({ error: err });
13            } else {
14                var numZip = parseInt(user.zipCode);
15                var fullName = "";
16                if (numZip >= 35004 && numZip <= 36925) {
17                    fullName = "Alabama";
18                }
19                else if (numZip >= 99501 && numZip <= 99950) {
20                    fullName = "Alaska";
21                }
22                else if ((numZip >= 71601 && numZip <= 75929) || numZip == 75502) {
23                    fullName = "Arkansas";
24                }
25                else if (numZip >= 85001 && numZip <= 86556) {
26                    fullName = "Arizona";
27                }
28                else if (numZip == 96799){
29                    fullName = "American Samoa";
30                }
31                else if (numZip >= 90001 && numZip <= 96162) {
32                    fullName = "California";
33                }
34                else if (numZip >= 80001 && numZip <= 81658) {
35                    fullName = "Colorado";
36                }
37                else if (numZip >= 6001 && numZip <= 6389) {
38                    fullName = "Connecticut";
39                }
40                else if (numZip >= 20001 && numZip <= 20799) {
41                    fullName = "Washington, D.C.";
42                }
43                else if (numZip >= 19701 && numZip <= 19980) {
44                    fullName = "Delaware";
45                }
46                else if (numZip >= 96941 && numZip <= 96944){
47                    fullName = "Federated States of Micronesia";
48                }
49                else if (numZip >= 32004 && numZip <= 34997) {
50                    fullName = "Florida";
51                }
52                else if ((numZip >= 30001 && numZip <= 31999) || numZip == 39901) {
53                    fullName = "Georgia";
54                }
55                else if (numZip >= 96910 && numZip <= 96932){
56                    fullName = "Guam";
57                }
58                else if (numZip >= 96701 && numZip <= 96898) {
59                    fullName = "Hawaii";
60                }
61                else if ((numZip >= 50001 && numZip <= 52809) || (numZip >= 68119 && numZip <= 68120)) {
62                    fullName = "Iowa";
63                }
64                else if (numZip >= 83201 && numZip <= 83876) {
65                    fullName = "Idaho";
66                }
67                else if (numZip >= 60001 && numZip <= 62999) {
68                    fullName = "Illinois";
69                }
70                else if (numZip >= 46001 && numZip <= 47997) {
71                    fullName = "Indiana";
72                }
73                else if (numZip >= 66002 && numZip <= 67954) {
74                    fullName = "Kansas";
75                }
76                else if (numZip >= 40003 && numZip <= 42788) {
77                    fullName = "Kentucky";
78                }
79                else if ((numZip >= 70001 && numZip <= 71232) || (numZip >= 71234 && numZip <= 71497)) {
80                    fullName = "Louisiana";
81                }
82                else if ((numZip >= 1001 && numZip <= 2791) || (numZip >= 5501 && numZip <= 5544)) {
83                    fullName = "Massachusetts";
84                }
85                else if (numZip == 20331 || (numZip >= 20335 && numZip <= 20797) || (numZip >= 20812 && numZip <= 21930)) {
86                    fullName = "Maryland";
87                }
88                else if (numZip >= 3901 && numZip <= 4992) {
89                    fullName = "Maine";
90                }
91                else if (numZip == 96960 || numZip == 96970) {
92                    fullName = "Marshall Islands";
93                }
94                else if (numZip >= 48001 && numZip <= 49971) {
95                    fullName = "Michigan";
96                }
97                else if (numZip >= 55001 && numZip <= 56763) {
98                    fullName = "Minnesota";
99                }
100                else if (numZip >= 63001 && numZip <= 65899) {
101                    fullName = "Missouri";
102                }
103                else if (numZip >= 96950 && numZip <= 96952) {
104                    fullName = "Northern Mariana Islands";
105                }
106                else if ((numZip >= 38601 && numZip <= 39776) || numZip == 71233) {
107                    fullName = "Mississippi";
108                }
109                else if (numZip >= 59001 && numZip <= 59937) {
110                    fullName = "Montana";
111                }
112                else if (numZip >= 27006 && numZip <= 28909) {
113                    fullName = "North Carolina";
114                }
115                else if (numZip >= 58001 && numZip <= 58856) {
116                    fullName = "North Dakota";
117                }
118                else if ((numZip >= 68001 && numZip <= 68118) || (numZip >= 68122 && numZip <= 69367)) {
119                    fullName = "Nebraska";
120                }
121                else if (numZip >= 3031 && numZip <= 3897) {
122                    fullName = "New Hampshire";
123                }
124                else if (numZip >= 7001 && numZip <= 8989) {
125                    fullName = "New Jersey";
126                }
127                else if (numZip >= 87001 && numZip <= 88441) {
128                    fullName = "New Mexico";
129                }
130                else if (numZip >= 88901 && numZip <= 89883) {
131                    fullName = "Nevada";
132                }
133                else if (numZip == 6390 || (numZip >= 10001 && numZip <= 14975)) {
134                    fullName = "New York";
135                }
136                else if (numZip >= 43001 && numZip <= 45999) {
137                    fullName = "Ohio";
138                }
139                else if ((numZip >= 73001 && numZip <= 73199) || (numZip >= 73401 && numZip <= 74966)) {
140                    fullName = "Oklahoma";
141                }
142                else if (numZip >= 97001 && numZip <= 97920) {
143                    fullName = "Oregon";
144                }
145                else if (numZip >= 15001 && numZip <= 19640) {
146                    fullName = "Pennsylvania";
147                }
148                else if ((numZip >= 601 && numZip <= 795) || (numZip >= 901 && numZip <= 988)) {
149                    fullName = "Puerto Rico";
150                }
151                else if (numZip >= 96939 && numZip <= 96940) {
152                    fullName = "Palau";
153                }
154                else if (numZip >= 2801 && numZip <= 2940) {
155                    fullName = "Rhode Island";
156                }
157                else if (numZip >= 29001 && numZip <= 29948) {
158                    fullName = "South Carolina";
159                }
160                else if (numZip >= 57001 && numZip <= 57799) {
161                    fullName = "South Dakota";
162                }
163                else if (numZip >= 37010 && numZip <= 38589) {
164                    fullName = "Tennessee";
165                }
166                else if (numZip == 73301 || (numZip >= 75001 && numZip <= 75501) || (numZip >= 75503 && numZip <= 79999) || (numZip >= 88510 && numZip <= 88589)) {
167                    fullName = "Texas";
168                }
169                else if (numZip >= 84001 && numZip <= 84784) {
170                    fullName = "Utah";
171                }
172                else if (numZip >= 20040 && numZip <= 24658) {
173                    fullName = "Virginia";
174                }
175                else if (numZip >= 801 && numZip <= 851){
176                    fullName = "Virgin Islands";
177                }
178                else if ((numZip >= 5001 && numZip <= 5495) || (numZip >= 5601 && numZip <= 5907)) {
179                    fullName = "Vermont";
180                }
181                else if (numZip >= 98001 && numZip <= 99403) {
182                    fullName = "Washington";
183                }
184                else if (numZip >= 53001 && numZip <= 54990) {
185                    fullName = "Wisconsin";
186                }
187                else if (numZip >= 24701 && numZip <= 26886) {
188                    fullName = "West Virginia";
189                }
190                else if (numZip >= 82001 && numZip <= 83128) {
191                    fullName = "Wyoming";
192                }
193                res.json({ state: fullName });
194            }
195        })
196    },
197
198    register: function(req, res) {
199        User.find({ email: req.body.email }, function(error, user) {
200            if (user.length >= 1) {
201                res.json({
202                    regEmailError: 'An account with that email already exists.  Please use a different email.',
203                    loggedIn: false
204                });
205            } else {
206                var newUser = new User({
207                    firstName: req.body.firstName,
208                    lastName: req.body.lastName,
209                    email: req.body.email,
210                    password: bcrypt.hashSync(req.body.password, bcrypt.genSaltSync(8)),
211                    userLevel: false,
212                });
213                newUser.reviews = [];
214                newUser.listings = [];
215                newUser.reservations = [];
216                newUser.conversations = [];
217                newUser.save(function(err, user) {
218                    if (err) {
219                        res.json({ error: err, loggedIn: false});
220                    } else {
221                        req.session.user = newUser;
222                        res.json({ user: newUser, loggedIn: true});
223                    }
224                });
225            }
226        });
227    },
228
229    login: function (req, res) {
230        User.find({email: req.body.email}, function(err, user){
231            if (err) {
232                res.json(err);
233            } else {
234                if (user.length > 0) {
235                    if (bcrypt.compareSync(req.body.password, user[0].password)) {
236                        req.session.user = user[0];
237                        res.json({user: user[0], loggedIn: true});
238                    } else {
239                        res.json({passwordError: 'Password is incorrect. Please try again', loggedIn: false});
240                    }
241                } else {
242                    res.json({emailError: 'Email not found, please register or try again',loggedIn: false});
243                }
244            }
245        });
246    },
247
248    logout: (req, res, next) => {
249        req.session.destroy();
250        res.json(true);
251    },
252
253    update: function (req, res) {
254        User.findById(req.params.id, function (err, user) {
255            if (err) {
256                console.log("Error", err);
257            } else {
258                user.firstName = req.body.firstName;
259                user.lastName = req.body.lastName;
260                user.email = req.body.email;
261                user.password = bcrypt.hashSync(req.body.password, bcrypt.genSaltSync(8));
262                // user.phoneNum = req.body.phoneNum;
263                user.save(function (err, user) {
264                    if (err) {
265                        res.json({ error: "Update attempt unsuccessful." });
266                    } else {
267                        res.json({ user: user });
268                    }
269                })
270            }
271        });
272    },
273
274    regHost: function (req, res) {
275        User.findById(req.params.id, function (err, user) {
276            if (err) {
277                res.json({ error: err });
278            } else {
279                if(user.userLevel != false){
280                    user.userLevel = true;
281                    user.save(function (err, user) {
282                        if (err) {
283                            res.json({ error: 'Unable to register as host. Please try again.' });
284                        } else {
285                            res.json({ user: user });
286                        }
287                    })
288                } else {
289                    res.json({ user: user });
290                }
291            }
292        });
293    },
294
295    current: function(req, res){
296        if (req.session.user){
297            var currentUser = req.session.user;
298            res.json({user: currentUser});
299        } else {
300            res.json({error: "Could not find user", 'user': req.session.user});
301        }
302    }
303}
Full Screen

downloadCode.js

Source: downloadCode.js Github

copy
1/*
2 * Gives the user a link to a zip file to download. The history of the code 
3 * is not included in the zip (there is no hidden /.storyteller directory).
4 */
5function zipAndDownloadCodeOnly() {
6    //create a new zip
7    const zip = new JSZip();
8    //add only the code to the zip
9    addCodeToZip(zip);
10    //cause the zip to be downloaded
11    downloadZip(zip);
12}
13/*
14 * Adds the code in the Ace editors to a zip file.
15 */
16function addCodeToZip(zip) {
17    //get the UL at the top of the fs view
18    const playbackViewOfFileSystem = document.getElementById('playbackViewOfFileSystem');
19    
20    //the root dir will be the one child of the UL, playbackViewOfFileSystem
21    if(playbackViewOfFileSystem.children.length === 1) {
22        //get the root dir LI
23        const dirLI = playbackViewOfFileSystem.children[0];
24        //get the dir id
25        const dirId = dirLI.getAttribute('data-directoryId');
26
27        //get all the active directories in the root (recursively)
28        const allDirs = getDirectoriesFromADirectory(dirId);
29
30        //create an entry in the zip for each dir (including empty dirs)
31        for(let i = 0;i < allDirs.length;i++) {
32            //get the dir path (JSZip doesn't like the leading / in paths so remove it)
33            const dir = allDirs[i];
34            const dirPath = dir.directoryPath.substring(1);
35            //add an entry for the dir (this is to preserve empty dirs)
36            zip.folder(dirPath);
37        }
38        
39        //get all the active files in the root (recursively)
40        const allFiles = getFilesFromADirectory(dirId);
41
42        //retrieve the text from the files and add an entry in the zip
43        for(let i = 0;i < allFiles.length;i++) {
44            //get the file path (JSZip doesn't like the leading / in paths so remove it)
45            const file = allFiles[i];
46            const editor = playbackData.editors[file.fileId];
47            
48            //get the file path and the text in the editor and add the file to the zip
49            const fileContents = editor.getSession().getValue();
50            const filePath = file.filePath.substring(1);
51            zip.file(filePath, fileContents);
52        }
53    }
54}
55/*
56 * Creates a zip file and a toast to display a download link.
57 */
58async function downloadZip(zip) {
59    //set the status of creating the zip message
60    const zipStatus = document.getElementById('zipStatus');
61    zipStatus.innerHTML = 'building zip file, this may take a while...';
62
63    //timing for debug purposes
64    //const t0 = performance.now();
65    
66    //create a blob representation of the zip
67    const blobbedZip = await zip.generateAsync({
68        type:'blob',
69        compression: 'DEFLATE',
70        compressionOptions: {
71            level: 9
72        }
73    });
74    
75    //const t1 = performance.now();
76    //console.log(`zip took: ${t1-t0} ms`);
77    
78    //clear out the zip status message
79    zipStatus.innerHTML = '';
80
81    //create a filename
82    let zipFileName = 'stProject.zip';
83
84    //create a downloadable blob
85    const blob = new Blob([blobbedZip], {type: 'application/zip'});
86    //create a url that holds the zip
87    const url = window.URL.createObjectURL(blob);
88
89    //create a link the user can click to download the code
90    const zipLink = document.getElementById('zipLink');
91    zipLink.innerHTML = zipFileName;
92    zipLink.href = url;
93    zipLink.download = zipFileName;
94
95    //create a toast to show the zip link download
96    const zipLinkToast = document.getElementById('zipLinkToast');
97    zipLinkToast.removeAttribute('style');
98    zipLinkToast.style.position = 'absolute';
99    zipLinkToast.style.zIndex = '100';
100
101    //sets the position of the toast to be under the download button
102    const codeDownloadButtonRectangle = document.getElementById('codeDownloadMenuButton').getBoundingClientRect();
103    zipLinkToast.style.top = codeDownloadButtonRectangle.y + codeDownloadButtonRectangle.height + 5 +'px';
104    zipLinkToast.style.left = codeDownloadButtonRectangle.left + 'px';
105    zipLinkToast.style.width = codeDownloadButtonRectangle.width + 'px';
106
107    //set a handler when the toast is closed
108    $('#zipLinkToast').on('hidden.bs.toast', function () {
109        //give back the resources for the zip file in memory
110        window.URL.revokeObjectURL(url);
111    });
112
113    //show the toast
114    $('#zipLinkToast').toast('show');
115}
116/*
117 * Recreates a storyteller project (with or without comments) and gives the 
118 * user a link to a zip file to download.
119 */
120async function zipCodeWithHistory(withComments) {
121    //create a new zip
122    const zip = new JSZip();
123    //add only the code to the zip
124    addCodeToZip(zip);
125    //add the data required to make this a storyeller project
126    await addStorytellerProjectHistoryToZip(playbackData.events.slice(0, playbackData.nextEventPosition), zip, withComments);
127    //cause the zip to be downloaded
128    downloadZip(zip);
129}
130/*
131 * Adds the required data to make this zip a true storyteller project that can 
132 * added to. 
133 */
134async function addStorytellerProjectHistoryToZip(events, zip, withComments) {
135    //add the required directories for a storyteller project
136    zip.folder('.storyteller');
137    zip.folder('.storyteller/comments');
138    zip.folder('.storyteller/comments/media');
139    zip.folder('.storyteller/comments/media/.tmp');
140    zip.folder('.storyteller/comments/media/audios');
141    zip.folder('.storyteller/comments/media/images');
142    zip.folder('.storyteller/comments/media/videos');
143    zip.folder('.storyteller/devs');
144    zip.folder('.storyteller/events');
145    zip.folder('.storyteller/events/intermediate');
146    zip.folder('.storyteller/fs');
147    zip.folder('.storyteller/project');
148
149    //data collected from the events up to the pause point
150    const zipPlaybackData = {
151        comments: {},
152