1 |
|
---|
2 |
|
---|
3 |
|
---|
4 |
|
---|
5 |
|
---|
6 |
|
---|
7 |
|
---|
8 |
|
---|
9 |
|
---|
10 |
|
---|
11 |
import os |
---|
12 |
import sys |
---|
13 |
import time |
---|
14 |
import getopt |
---|
15 |
import string |
---|
16 |
import ftplib |
---|
17 |
from fnmatch import fnmatch |
---|
18 |
|
---|
19 |
usage_msg = """ |
---|
20 |
usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] |
---|
21 |
[-l username [-p passwd [-a account]]] |
---|
22 |
hostname [remotedir [localdir]] |
---|
23 |
-v: verbose |
---|
24 |
-q: quiet |
---|
25 |
-i: interactive mode |
---|
26 |
-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o') |
---|
27 |
-n: don't log in |
---|
28 |
-r: remove files no longer pertinent |
---|
29 |
-l username [-p passwd [-a account]]: login info (default anonymous ftp) |
---|
30 |
-s pat: skip files matching pattern |
---|
31 |
hostname: remote host |
---|
32 |
remotedir: remote directory (default initial) |
---|
33 |
localdir: local directory (default current) |
---|
34 |
""" |
---|
35 |
def usage(*args): |
---|
36 |
sys.stdout = sys.stderr |
---|
37 |
for msg in args: print msg |
---|
38 |
print usage_msg |
---|
39 |
sys.exit(2) |
---|
40 |
|
---|
41 |
verbose = 1 |
---|
42 |
interactive = 0 |
---|
43 |
mac = 0 |
---|
44 |
rmok = 0 |
---|
45 |
nologin = 0 |
---|
46 |
skippats = ['.', '..', '.mirrorinfo'] |
---|
47 |
|
---|
48 |
def main(): |
---|
49 |
global verbose, interactive, mac, rmok, nologin |
---|
50 |
try: |
---|
51 |
opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') |
---|
52 |
except getopt.error, msg: |
---|
53 |
usage(msg) |
---|
54 |
login = '' |
---|
55 |
passwd = '' |
---|
56 |
account = '' |
---|
57 |
for o, a in opts: |
---|
58 |
if o == '-l': login = a |
---|
59 |
if o == '-p': passwd = a |
---|
60 |
if o == '-a': account = a |
---|
61 |
if o == '-v': verbose = verbose + 1 |
---|
62 |
if o == '-q': verbose = 0 |
---|
63 |
if o == '-i': interactive = 1 |
---|
64 |
if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') |
---|
65 |
if o == '-n': nologin = 1 |
---|
66 |
if o == '-r': rmok = 1 |
---|
67 |
if o == '-s': skippats.append(a) |
---|
68 |
if not args: usage('hostname missing') |
---|
69 |
host = args[0] |
---|
70 |
remotedir = '' |
---|
71 |
localdir = '' |
---|
72 |
if args[1:]: |
---|
73 |
remotedir = args[1] |
---|
74 |
if args[2:]: |
---|
75 |
localdir = args[2] |
---|
76 |
if args[3:]: usage('too many arguments') |
---|
77 |
|
---|
78 |
f = ftplib.FTP() |
---|
79 |
if verbose: print 'Connecting to %s...' % host |
---|
80 |
f.connect(host) |
---|
81 |
if not nologin: |
---|
82 |
if verbose: |
---|
83 |
print 'Logging in as %s...' % (login or 'anonymous') |
---|
84 |
f.login(login, passwd, account) |
---|
85 |
if verbose: print 'OK.' |
---|
86 |
pwd = f.pwd() |
---|
87 |
if verbose > 1: print 'PWD =', `pwd` |
---|
88 |
if remotedir: |
---|
89 |
if verbose > 1: print 'cwd(%s)' % `remotedir` |
---|
90 |
f.cwd(remotedir) |
---|
91 |
if verbose > 1: print 'OK.' |
---|
92 |
pwd = f.pwd() |
---|
93 |
if verbose > 1: print 'PWD =', `pwd` |
---|
94 |
|
---|
95 |
mirrorsubdir(f, localdir) |
---|
96 |
|
---|
97 |
def mirrorsubdir(f, localdir): |
---|
98 |
pwd = f.pwd() |
---|
99 |
if localdir and not os.path.isdir(localdir): |
---|
100 |
if verbose: print 'Creating local directory', localdir |
---|
101 |
try: |
---|
102 |
makedir(localdir) |
---|
103 |
except os.error, msg: |
---|
104 |
print "Failed to establish local directory", localdir |
---|
105 |
return |
---|
106 |
infofilename = os.path.join(localdir, '.mirrorinfo') |
---|
107 |
try: |
---|
108 |
text = open(infofilename, 'r').read() |
---|
109 |
except IOError, msg: |
---|
110 |
text = '{}' |
---|
111 |
try: |
---|
112 |
info = eval(text) |
---|
113 |
except (SyntaxError, NameError): |
---|
114 |
print 'Bad mirror info in %s' % infofilename |
---|
115 |
info = {} |
---|
116 |
subdirs = [] |
---|
117 |
listing = [] |
---|
118 |
if verbose: print 'Listing remote directory %s...' % pwd |
---|
119 |
f.retrlines('LIST', listing.append) |
---|
120 |
for line in listing: |
---|
121 |
if verbose > 1: print '-->', `line` |
---|
122 |
if mac: |
---|
123 |
|
---|
124 |
|
---|
125 |
filename = string.strip(line) |
---|
126 |
mode = '-' |
---|
127 |
if filename[-1:] == '/': |
---|
128 |
filename = filename[:-1] |
---|
129 |
mode = 'd' |
---|
130 |
infostuff = '' |
---|
131 |
else: |
---|
132 |
|
---|
133 |
words = string.split(line) |
---|
134 |
if len(words) < 6: |
---|
135 |
if verbose > 1: print 'Skipping short line' |
---|
136 |
continue |
---|
137 |
if words[-2] == '->': |
---|
138 |
if verbose > 1: |
---|
139 |
print 'Skipping symbolic link %s -> %s' % \ |
---|
140 |
(words[-3], words[-1]) |
---|
141 |
continue |
---|
142 |
filename = words[-1] |
---|
143 |
infostuff = words[-5:-1] |
---|
144 |
mode = words[0] |
---|
145 |
skip = 0 |
---|
146 |
for pat in skippats: |
---|
147 |
if fnmatch(filename, pat): |
---|
148 |
if verbose > 1: |
---|
149 |
print 'Skip pattern', pat, |
---|
150 |
print 'matches', filename |
---|
151 |
skip = 1 |
---|
152 |
break |
---|
153 |
if skip: |
---|
154 |
continue |
---|
155 |
if mode[0] == 'd': |
---|
156 |
if verbose > 1: |
---|
157 |
print 'Remembering subdirectory', filename |
---|
158 |
subdirs.append(filename) |
---|
159 |
continue |
---|
160 |
if info.has_key(filename) and info[filename] == infostuff: |
---|
161 |
if verbose > 1: |
---|
162 |
print 'Already have this version of', filename |
---|
163 |
continue |
---|
164 |
fullname = os.path.join(localdir, filename) |
---|
165 |
tempname = os.path.join(localdir, '@'+filename) |
---|
166 |
if interactive: |
---|
167 |
doit = askabout('file', filename, pwd) |
---|
168 |
if not doit: |
---|
169 |
if not info.has_key(filename): |
---|
170 |
info[filename] = 'Not retrieved' |
---|
171 |
continue |
---|
172 |
try: |
---|
173 |
os.unlink(tempname) |
---|
174 |
except os.error: |
---|
175 |
pass |
---|
176 |
try: |
---|
177 |
fp = open(tempname, 'w') |
---|
178 |
except IOError, msg: |
---|
179 |
print "Can't create %s: %s" % (tempname, str(msg)) |
---|
180 |
continue |
---|
181 |
if verbose: |
---|
182 |
print 'Retrieving %s from %s as %s...' % \ |
---|
183 |
(filename, pwd, fullname) |
---|
184 |
if verbose: |
---|
185 |
fp1 = LoggingFile(fp, 1024, sys.stdout) |
---|
186 |
else: |
---|
187 |
fp1 = fp |
---|
188 |
t0 = time.time() |
---|
189 |
try: |
---|
190 |
f.retrbinary('RETR ' + filename, fp1.write, 8*1024) |
---|
191 |
except ftplib.error_perm, msg: |
---|
192 |
print msg |
---|
193 |
t1 = time.time() |
---|
194 |
bytes = fp.tell() |
---|
195 |
fp.close() |
---|
196 |
if fp1 != fp: |
---|
197 |
fp1.close() |
---|
198 |
try: |
---|
199 |
os.rename(tempname, fullname) |
---|
200 |
except os.error, msg: |
---|
201 |
print "Can't rename %s to %s: %s" % (tempname, |
---|
202 |
fullname, |
---|
203 |
str(msg)) |
---|
204 |
continue |
---|
205 |
info[filename] = infostuff |
---|
206 |
writedict(info, infofilename) |
---|
207 |
if verbose: |
---|
208 |
dt = t1 - t0 |
---|
209 |
kbytes = bytes / 1024.0 |
---|
210 |
print int(round(kbytes)), |
---|
211 |
print 'Kbytes in', |
---|
212 |
print int(round(dt)), |
---|
213 |
print 'seconds', |
---|
214 |
if t1 > t0: |
---|
215 |
print '(~%d Kbytes/sec)' % \ |
---|
216 |
int(round(kbytes/dt),) |
---|
217 |
print |
---|
218 |
|
---|
219 |
|
---|
220 |
try: |
---|
221 |
if not localdir: names = os.listdir(os.curdir) |
---|
222 |
else: names = os.listdir(localdir) |
---|
223 |
except os.error: |
---|
224 |
names = [] |
---|
225 |
for name in names: |
---|
226 |
if name[0] == '.' or info.has_key(name) or name in subdirs: |
---|
227 |
continue |
---|
228 |
fullname = os.path.join(localdir, name) |
---|
229 |
if not rmok: |
---|
230 |
if verbose: |
---|
231 |
print 'Local file', fullname, |
---|
232 |
print 'is no longer pertinent' |
---|
233 |
continue |
---|
234 |
if verbose: print 'Removing local file', fullname |
---|
235 |
try: |
---|
236 |
os.unlink(fullname) |
---|
237 |
except os.error, msg: |
---|
238 |
print "Can't remove local file %s: %s" % \ |
---|
239 |
(fullname, str(msg)) |
---|
240 |
|
---|
241 |
|
---|
242 |
for subdir in subdirs: |
---|
243 |
if interactive: |
---|
244 |
doit = askabout('subdirectory', subdir, pwd) |
---|
245 |
if not doit: continue |
---|
246 |
if verbose: print 'Processing subdirectory', subdir |
---|
247 |
localsubdir = os.path.join(localdir, subdir) |
---|
248 |
pwd = f.pwd() |
---|
249 |
if verbose > 1: |
---|
250 |
print 'Remote directory now:', pwd |
---|
251 |
print 'Remote cwd', subdir |
---|
252 |
try: |
---|
253 |
f.cwd(subdir) |
---|
254 |
except ftplib.error_perm, msg: |
---|
255 |
print "Can't chdir to", subdir, ":", msg |
---|
256 |
else: |
---|
257 |
if verbose: print 'Mirroring as', localsubdir |
---|
258 |
mirrorsubdir(f, localsubdir) |
---|
259 |
if verbose > 1: print 'Remote cwd ..' |
---|
260 |
f.cwd('..') |
---|
261 |
newpwd = f.pwd() |
---|
262 |
if newpwd != pwd: |
---|
263 |
print 'Ended up in wrong directory after cd + cd ..' |
---|
264 |
print 'Giving up now.' |
---|
265 |
break |
---|
266 |
else: |
---|
267 |
if verbose > 1: print 'OK.' |
---|
268 |
|
---|
269 |
|
---|
270 |
class LoggingFile: |
---|
271 |
def __init__(self, fp, blocksize, outfp): |
---|
272 |
self.fp = fp |
---|
273 |
self.bytes = 0 |
---|
274 |
self.hashes = 0 |
---|
275 |
self.blocksize = blocksize |
---|
276 |
self.outfp = outfp |
---|
277 |
def write(self, data): |
---|
278 |
self.bytes = self.bytes + len(data) |
---|
279 |
hashes = int(self.bytes) / self.blocksize |
---|
280 |
while hashes > self.hashes: |
---|
281 |
self.outfp.write('#') |
---|
282 |
self.outfp.flush() |
---|
283 |
self.hashes = self.hashes + 1 |
---|
284 |
self.fp.write(data) |
---|
285 |
def close(self): |
---|
286 |
self.outfp.write('\n') |
---|
287 |
|
---|
288 |
|
---|
289 |
def askabout(filetype, filename, pwd): |
---|
290 |
prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) |
---|
291 |
while 1: |
---|
292 |
reply = string.lower(string.strip(raw_input(prompt))) |
---|
293 |
if reply in ['y', 'ye', 'yes']: |
---|
294 |
return 1 |
---|
295 |
if reply in ['', 'n', 'no', 'nop', 'nope']: |
---|
296 |
return 0 |
---|
297 |
print 'Please answer yes or no.' |
---|
298 |
|
---|
299 |
|
---|
300 |
|
---|
301 |
def makedir(pathname): |
---|
302 |
if os.path.isdir(pathname): |
---|
303 |
return |
---|
304 |
dirname = os.path.dirname(pathname) |
---|
305 |
if dirname: makedir(dirname) |
---|
306 |
os.mkdir(pathname, 0777) |
---|
307 |
|
---|
308 |
|
---|
309 |
|
---|
310 |
def writedict(dict, filename): |
---|
311 |
fp = open(filename, 'w') |
---|
312 |
fp.write('{\n') |
---|
313 |
for key, value in dict.items(): |
---|
314 |
fp.write('%s: %s,\n' % (`key`, `value`)) |
---|
315 |
fp.write('}\n') |
---|
316 |
fp.close() |
---|
317 |
|
---|
318 |
main() |
---|