/stdhome

To get this branch, use:
bzr branch http://bzr.ed.am/stdhome

« back to all changes in this revision

Viewing changes to lib/stdhome/deployment.py

  • Committer: Tim Marston
  • Date: 2014-02-26 19:10:31 UTC
  • Revision ID: tim@ed.am-20140226191031-elcqy5j09h2syn2j
moved copy-in, copy-out and deployment conflict checking to a set of "walkers";
bzr vcs back-end now parses affected files during update; deployment state now
includes affected files

Show diffs side-by-side

added added

removed removed

Lines of Context:
19
19
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
20
20
 
21
21
 
22
 
import os, re, shutil, filecmp
 
22
import os, re, shutil, filecmp, json
23
23
import the, util
24
 
 
 
24
from walker.copy_in_walker import CopyInWalker
 
25
from walker.conflict_walker import ConflictWalker
 
26
from walker.copy_out_walker import CopyOutWalker
25
27
 
26
28
class Deployment:
27
29
 
43
45
        # of the state of the filesystem.
44
46
                self.deploy_files = None
45
47
 
 
48
                # list of files that were changed by a recent vcs update (so only these
 
49
                # need to be checked for deployment conflicts or copied-out)
 
50
                self.updated_files = None
 
51
 
46
52
                # do we have a repo?
47
53
                if not os.path.exists( the.repo.full_dir ): return
48
54
 
49
55
                # if no file list exists, we're done
50
56
                file = os.path.join( the.full_mddir, "deploy.%s" % the.repo.name )
51
57
                if not os.path.isfile( file ):
52
 
                        if the.verbose: print "no deployment state found"
53
58
                        return
54
59
 
55
60
                # read the file list
56
61
                if the.verbose: print "loading deployment state"
57
62
                f = open( file, 'r' )
58
 
                self.deploy_files = f.read().splitlines()
 
63
                state = json.loads( f.read() )
 
64
 
 
65
                # unpack deployment state
 
66
                self.deploy_files = state['deploy_files'];
 
67
                self.updated_files = state['updated_files'];
59
68
 
60
69
 
61
70
        def save_deployment_state( self ):
65
74
                if not os.path.isdir( the.full_mddir ):
66
75
                        os.mkdir( the.full_mddir )
67
76
 
 
77
                # pack deployment state
 
78
                state = {
 
79
                        'deploy_files': self.deploy_files,
 
80
                        'updated_files': self.updated_files,
 
81
                }
 
82
 
68
83
                # create file
69
84
                file = os.path.join( the.full_mddir, "deploy.%s" % the.repo.name )
70
85
                f = open( file, 'w' )
71
 
                f.write( '\n'.join( self.deploy_files ) + '\n' )
 
86
                f.write( json.dumps( state ) );
72
87
 
73
88
 
74
89
        def remove_deployment_state( self ):
75
90
 
76
 
                # delete it, if it exists
 
91
                # check it exists
77
92
                file = os.path.join( the.full_mddir, "deploy.%s" % the.repo.name )
78
93
                if( os.path.isfile( file ) ):
 
94
 
 
95
                        # delete it
 
96
                        if the.verbose: print "removing deployment state"
79
97
                        os.unlink( file )
80
98
 
81
99
 
92
110
                                raise self.DeploymentOngoing( True )
93
111
 
94
112
 
95
 
        def confirm( self, message ):
96
 
                # TODO: write interactive confirmation routine
97
 
                return False
98
 
 
99
 
 
100
 
        def walk_repo( self, dir_func = None, file_func = None, link_func = None,
101
 
                                   relative_dir = '', ignore_errors = False ):
102
 
                """Walk through all files and directories in the repo, passing the relative path
103
 
                to each to the provided functions.
104
 
                """
105
 
                full_dir = os.path.join( the.repo.full_dir, relative_dir )
106
 
                for file in os.listdir( full_dir ):
107
 
                        relative_file = os.path.join( relative_dir, file )
108
 
 
109
 
                        # skip some stuff
110
 
                        if relative_file == '.bzr': continue
111
 
 
112
 
                        repo_file = os.path.join( the.repo.full_dir, relative_file )
113
 
                        fs_file = os.path.join( the.full_fsdir, relative_file )
114
 
                        if os.path.islink( repo_file ):
115
 
                                if link_func is not None:
116
 
                                        link_func( relative_file, repo_file, fs_file )
117
 
                        elif os.path.isfile( repo_file ):
118
 
                                if file_func is not None:
119
 
                                        file_func( relative_file, repo_file, fs_file )
120
 
                        elif os.path.isdir( repo_file ):
121
 
                                if dir_func is not None:
122
 
                                        dir_func( relative_path, repo_file, fs_file )
123
 
                                # recurse in directories
124
 
                                self.walk_repo( dir_func, file_func, link_func, relative_dir,
125
 
                                                                ignore_errors )
126
 
                        elif not ignore_errors:
127
 
                                raise RuntimeError(
128
 
                                        'repo contains unknown/missing entity' )
129
 
 
130
 
 
131
113
        def copy_in( self ):
132
114
 
133
 
                def copy_in_dir( relative_file, dst, src ):
134
 
                        self.deploy_files.append( relative_file )
135
 
 
136
 
                        # if src doesn't exist, delete it from the repo
137
 
                        if not os.path.lexists( src ):
138
 
                                if the.verbose: print " _<d " + relative_file
139
 
                                shutil.rmtree( dst )
140
 
 
141
 
                        # if src is a directory, copy permissions, as necessary
142
 
                        elif os.path.isdir( src ):
143
 
                                # TODO: should check permissions and only do as necessary
144
 
                                if the.verbose: print " d<d " + relative_file
145
 
                                shutil.copystat( src, dst )
146
 
 
147
 
                        # TODO: serious differences in between ~/ and repo (e.g., files in
148
 
                        # one that are directories in the other) should be ignored (e.g.,
149
 
                        # not copied-in).  And the stuff that is ignored during copy-in
150
 
                        # should also be ignored during copy-out and must not be added to
151
 
                        # the deploy_files list.  Since these ignored files/directories are
152
 
                        # transparent to the user, they should have to explicitly permit
153
 
                        # them via an ignore file (e.g., ~/.stdhome/.ignore, akin to bzr's
154
 
                        # .bzrignore file).  If these serious differences are not matched by
155
 
                        # the ignore file, an error should show (which will requie a
156
 
                        # separate "check" walk of the repo, as is done in copy_out).
157
 
                        else:
158
 
                                raise self.Conflict( "%s differs too severely from the repo" %
159
 
                                                                         os.path.join( the.fsdir, relative_file ) )
160
 
 
161
 
                def copy_in_file( relative_file, dst, src ):
162
 
                        self.deploy_files.append( relative_file )
163
 
 
164
 
                        # if src doesn't exist, delete it in the repo
165
 
                        if not os.path.lexists( src ):
166
 
                                if the.verbose: print " ?<f " + relative_file
167
 
                                os.unlink( dst )
168
 
 
169
 
                        # if src is a symlink, replace it in repo
170
 
                        elif os.path.islink( src ):
171
 
                                if the.verbose: print " l<f " + relative_file
172
 
                                os.unlink( dst )
173
 
                                os.symlink( os.readlink( src ), dst )
174
 
 
175
 
                        # if src is a file, replace it if different
176
 
                        elif os.path.isfile( src ):
177
 
                                if not filecmp.cmp( src, dst ):
178
 
                                        if the.verbose: print " f<f " + relative_file
179
 
                                        os.unlink( dst )
180
 
                                        shutil.copy( src, dst )
181
 
                                        shutil.copystat( src, dst )
182
 
                                else:
183
 
                                        if the.verbose: print " f=f " + relative_file
184
 
 
185
 
                        # TODO: serious differences in between ~/ and repo (e.g., files in
186
 
                        # one that are directories in the other) should be ignored (e.g.,
187
 
                        # not copied-in).  And the stuff that is ignored during copy-in
188
 
                        # should also be ignored during copy-out and must not be added to
189
 
                        # the deploy_files list.  Since these ignored files/directories are
190
 
                        # transparent to the user, they should have to explicitly permit
191
 
                        # them via an ignore file (e.g., ~/.stdhome/.ignore, akin to bzr's
192
 
                        # .bzrignore file).  If these serious differences are not matched by
193
 
                        # the ignore file, an error should show (which will requie a
194
 
                        # separate "check" walk of the repo, as is done in copy_out).
195
 
                        else:
196
 
                                raise self.Conflict( "%s differs too severely from the repo" %
197
 
                                                                         os.path.join( the.fsdir, relative_file ) )
198
 
 
199
 
                def copy_in_link( relative_file, dst, src ):
200
 
                        self.deploy_files.append( relative_file )
201
 
 
202
 
                        # if src doesn't exist, delete it in the repo
203
 
                        if not os.path.lexists( src ):
204
 
                                if the.verbose: print " _<l " + relative_file
205
 
                                os.unlink( dst )
206
 
 
207
 
                        # if src is a symlink, replace it if different in repo
208
 
                        elif os.path.islink( src ):
209
 
                                if os.readlink( src ) != os.readlink( dst ):
210
 
                                        if the.verbose: print " l<l " + relative_file
211
 
                                        os.unlink( dst )
212
 
                                        os.symlink( os.readlink( src ), dst )
213
 
                                else:
214
 
                                        if the.verbose: print " l=l " + relative_file
215
 
 
216
 
                        # if src is a file, replace it in repo
217
 
                        elif os.path.isfile( src ):
218
 
                                if the.verbose: print " f<l " + relative_file
219
 
                                os.unlink( dst )
220
 
                                shutil.copy( src, dst )
221
 
                                shutil.copystat( src, dst )
222
 
 
223
 
                        # TODO: serious differences in between ~/ and repo (e.g., files in
224
 
                        # one that are directories in the other) should be ignored (e.g.,
225
 
                        # not copied-in).  And the stuff that is ignored during copy-in
226
 
                        # should also be ignored during copy-out and must not be added to
227
 
                        # the deploy_files list.  Since these ignored files/directories are
228
 
                        # transparent to the user, they should have to explicitly permit
229
 
                        # them via an ignore file (e.g., ~/.stdhome/.ignore, akin to bzr's
230
 
                        # .bzrignore file).  If these serious differences are not matched by
231
 
                        # the ignore file, an error should show (which will requie a
232
 
                        # separate "check" walk of the repo, as is done in copy_out).
233
 
                        else:
234
 
                                raise self.Conflict( "%s differs too severely from the repo" %
235
 
                                                                         os.path.join( the.fsdir, relative_file ) )
236
 
 
237
115
                # check we don't already have a file list
238
116
                self.check_ongoing( False )
239
117
 
240
 
                # new file list
241
 
                self.deploy_files = list()
242
 
 
243
 
                # if the repo doesn't exist, we're done
244
 
                if not os.path.exists( the.repo.full_dir ): return
245
 
 
246
 
                # copy in
247
 
                if the.verbose: print "importing files"
248
 
                self.walk_repo( dir_func = copy_in_dir, file_func = copy_in_file,
249
 
                                           link_func = copy_in_link, ignore_errors = True )
 
118
                # if the repo doesn't exist, we have an empty file list
 
119
                if not os.path.exists( the.repo.full_dir ):
 
120
                        self.deploy_files = list()
 
121
                else:
 
122
                        # copy in
 
123
                        if the.verbose: print "importing files..."
 
124
                        walker = CopyInWalker()
 
125
                        walker.walk()
 
126
                        if( walker.changed ):
 
127
                                raise self.CopyInConflicts( walker.changed )
 
128
                        self.deploy_files = walker.walk_list
250
129
 
251
130
                # save state
252
131
                self.save_deployment_state()
253
132
 
254
133
 
 
134
        def get_conflicts( self, updated_files = None ):
 
135
 
 
136
                # check we have a file list
 
137
                self.check_ongoing( True )
 
138
 
 
139
                # set updated files
 
140
                if updated_files is not None:
 
141
                        self.updated_files = updated_files
 
142
                        self.save_deployment_state()
 
143
 
 
144
                # check for deployment conflicts
 
145
                walker = ConflictWalker( self.deploy_files, self.updated_files )
 
146
                walker.walk()
 
147
 
 
148
                self.conflicts_checked = True
 
149
                return walker.changed
 
150
 
 
151
 
255
152
        def copy_out( self ):
256
153
 
257
 
                def copy_out_dir( relative_file, src, dst ):
258
 
 
259
 
                        # if dst doesn't exist, create it
260
 
                        if not os.path.lexists( dst ):
261
 
                                if the.verbose: print " d>_ " + relative_file
262
 
                                os.mkdir( dst )
263
 
                                shutil.copystat( src, dst )
264
 
 
265
 
                        # if dst is a file/symlink, replace it
266
 
                        elif os.path.isfile( dst ):
267
 
                                if the.verbose: print " d>f " + relative_file
268
 
                                os.unlink( dst )
269
 
                                os.mkdir( dst )
270
 
                                shutil.copystat( src, dst )
271
 
 
272
 
                        # if dst is a directory, copy permissions as required
273
 
                        elif os.path.isdir( dst ):
274
 
                                # TODO: should check permission and only do as necessary
275
 
                                if the.verbose: print " d>d " + relative_file
276
 
                                shutil.copystat( src, dst )
277
 
 
278
 
                        else:
279
 
                                raise NotImplementedError()
280
 
 
281
 
                def copy_out_file( relative_file, src, dst ):
282
 
 
283
 
                        # if dst doesn't exist, copy
284
 
                        if not os.path.lexists( dst ):
285
 
                                if the.verbose: print " f>_ " + relative_file
286
 
 
287
 
                                shutil.copy( src, dst )
288
 
                                shutil.copystat( src, dst )
289
 
 
290
 
                        # if dst is a symlink, replace it
291
 
                        elif os.path.islink( dst ):
292
 
                                if the.verbose: print " f>l " + relative_file
293
 
                                os.unlink( dst )
294
 
                                shutil.copy( src, dst )
295
 
                                shutil.copystat( src, dst )
296
 
 
297
 
                        # if dst is a file, replace it if different
298
 
                        elif os.path.isfile( dst ):
299
 
                                if not filecmp.cmp( src, dst ):
300
 
                                        if the.verbose: print " f>f " + relative_file
301
 
                                        os.unlink( dst )
302
 
                                        shutil.copy( src, dst )
303
 
                                        shutil.copystat( src, dst )
304
 
                                else:
305
 
                                        if the.verbose: print " f=f " + relative_file
306
 
 
307
 
                        # if dst is a directory, replace it
308
 
                        elif os.path.isdir( dst ):
309
 
                                if the.verbose: print " f>d " + relative_file
310
 
                                shutil.rmtree( dst )
311
 
                                shutil.copy( src, dst )
312
 
                                shutil.copystat( src, dst )
313
 
 
314
 
                        else:
315
 
                                raise NotImplementedError()
316
 
 
317
 
                def copy_out_link( relative_file, src, dst ):
318
 
 
319
 
                        # if dst doesn't exist, copy
320
 
                        if not os.path.lexists( dst ):
321
 
                                if the.verbose: print " l>_ " + relative_file
322
 
                                os.symlink( os.readlink( src ), dst )
323
 
 
324
 
                        # if dst is a symlink, replace it if different
325
 
                        elif os.path.islink( dst ):
326
 
                                if os.readlink( src ) != os.readlink( dst ):
327
 
                                        if the.verbose: print " l>l " + relative_file
328
 
                                        os.unlink( dst )
329
 
                                        os.symlink( os.readlink( src ), dst )
330
 
                                else:
331
 
                                        if the.verbose: print " l=l " + relative_file
332
 
 
333
 
                        # if dst is a file, replace it
334
 
                        elif os.path.isfile( dst ):
335
 
                                if the.verbose: print " l>f " + relative_file
336
 
                                os.unlink( dst )
337
 
                                os.symlink( os.readlink( src ), dst )
338
 
 
339
 
                        # if dst is a directory, replace it
340
 
                        elif os.path.isdir( dst ):
341
 
                                if the.verbose: print " l>d " + relative_file
342
 
                                shutil.rmtree( dst )
343
 
                                os.symlink( os.readlink( src ), dst )
344
 
 
345
 
                        else:
346
 
                                raise NotImplementedError()
347
 
 
348
154
                # check we have a file list
349
155
                self.check_ongoing( True )
350
156
 
351
 
                # we should already have handled conflicts
 
157
                # check that deployment conflicts have been checked-for
352
158
                if not self.conflicts_checked:
353
 
                        raise RuntimeError(
354
 
                                'logic error: conflicts should have been checked!' )
 
159
                        raise RuntimeError('logic error: deployment conflicts unchecked' )
355
160
 
356
161
                # copy out
357
 
                if the.verbose: print "exporting files"
358
 
                self.walk_repo( dir_func = copy_out_dir, file_func = copy_out_file,
359
 
                                           link_func = copy_out_link, ignore_errors = True )
 
162
                if the.verbose: print "exporting files..."
 
163
                walker = CopyOutWalker( self.updated_files )
 
164
                walker.walk()
360
165
 
361
166
                # clear state
362
167
                self.remove_deployment_state()
363
168
 
364
169
 
365
 
        def get_conflicts( self ):
366
 
 
367
 
                def check_dir( relative_file, src, dst ):
368
 
 
369
 
                        # files that existed at copy-in can be copied-out and overwritten
370
 
                        if self.deploy_files is not None and \
371
 
                           relative_file in self.deploy_files: return
372
 
 
373
 
                        # files that don't exist in the filesystem can be copied-out
374
 
                        if not os.path.lexists( dst ): return
375
 
 
376
 
                        # accept/reject existing files
377
 
                        if os.path.islink( dst ):
378
 
                                self.files.append( "%s already exists (as a symlink)" %
379
 
                                                                   os.path.join( the.fsdir, relative_file ) )
380
 
                        elif os.path.isfile( dst ):
381
 
                                self.files.append( "%s already exists (as a file)" %
382
 
                                                                   os.path.join( the.fsdir, relative_file ) )
383
 
                        elif os.path.isdir( dst ):
384
 
                                return
385
 
                        else:
386
 
                                self.files.append( "%s already exists" %
387
 
                                                                   os.path.join( the.fsdir, relative_file ) )
388
 
 
389
 
                def check_file( relative_file, src, dst ):
390
 
 
391
 
                        # files that existed at copy-in can be copied-out and overwritten
392
 
                        if self.deploy_files is not None and \
393
 
                           relative_file in self.deploy_files: return
394
 
 
395
 
                        # files that don't exist in the filesystem can be copied-out
396
 
                        if not os.path.lexists( dst ): return
397
 
 
398
 
                        # accept/reject existing files
399
 
                        if os.path.isfile( dst ):
400
 
                                self.files.append( "%s already exists" %
401
 
                                                                   os.path.join( the.fsdir, relative_file ) )
402
 
                        elif os.path.isdir( dst ):
403
 
                                self.files.append( "%s already exists (as a directory)" %
404
 
                                                                   os.path.join( the.fsdir, relative_file ) )
405
 
                        else:
406
 
                                self.files.append( "%s already exists" %
407
 
                                                                   os.path.join( the.fsdir, relative_file ) )
408
 
 
409
 
                self.conflicts_checked = True
410
 
 
411
 
                # check for conflicts
412
 
                self.files = list()
413
 
                self.walk_repo( dir_func = check_dir, file_func = check_file,
414
 
                                           link_func = check_file )
415
 
                return self.files
416
 
 
417
 
 
418
170
        class DeploymentOngoing( the.program.FatalError ):
419
171
 
420
172
                def __init__( self, ongoing ):
424
176
                                self.msg = "there is no ongoing deployment"
425
177
 
426
178
 
427
 
        class Conflict( Exception ):
 
179
        class CopyInConflicts( the.program.FatalError ):
428
180
 
429
 
                def __init__( self, message ):
430
 
                        self.msg = message
 
181
                def __init__( self, conflicts ):
 
182
                        self.conflicts = conflicts