1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
|
#!/usr/bin/env python
# Copyright 2014-2016 Gentoo Authors; Distributed under the GPL v2
from __future__ import unicode_literals
import io
import os
import sys
import time
from xml.sax.saxutils import escape as escape_xml
def usage():
sys.stderr.write(("usage: %s <failure log> <success log> "
"<scheduled deletion log> <log date format> <distfiles dir> "
"<output xml file> [whitelist dir]\n") % \
os.path.basename(sys.argv[0]))
sys.stderr.flush()
if len(sys.argv) < 7 or len(sys.argv) > 8:
usage()
sys.exit(1)
if not os.access(sys.argv[1], os.R_OK):
sys.stderr.write("couldn't read %s\n" % sys.argv[1])
sys.exit(2)
fail_f=open(sys.argv[1], "r")
suc_f=open(sys.argv[2],"r")
if not os.path.isfile(sys.argv[3]):
sys.stderr.write("%s isn't a file.\n" % sys.argv[3])
sys.exit(3)
scheduled_deletions_log = sys.argv[3]
log_date_format = sys.argv[4]
try:
outf = io.open(sys.argv[6], mode='w', encoding="utf_8")
except OSError as e:
sys.stderr.write("failed opening out file; lack write "
"perms, dir exist? %s\n" % (e,))
sys.exit(4)
def grab_whitelists(whitelists_dir):
whitelists = {}
for x in os.listdir(whitelists_dir):
if x[:1] == ".":
continue
x = os.path.join(whitelists_dir, x)
if not os.path.isfile(x):
continue
whitelists[x] = []
with io.open(x, mode='r', encoding='utf_8') as f:
for entry in f:
entry = entry.lstrip().rstrip()
if len(entry) == 0 or entry.startswith("#"):
continue
whitelists[x].append(entry.lstrip().rstrip())
if not whitelists[x]:
del whitelists[x]
return whitelists
if len(sys.argv) >= 8:
whitelist = grab_whitelists(sys.argv[7])
else:
whitelist = {}
failed_ebuilds={}
for l in fail_f:
try:
cpv, uri, reason = l.split("\t")
except ValueError:
continue
cpv = cpv.lstrip().rstrip()
uri = uri.lstrip().rstrip()
reason = reason.lstrip().rstrip()
#cpv, file, reason
failed_ebuilds.setdefault(cpv, []).append((uri, reason))
fail_f.close()
removed_files = {}
added_files = {}
for l in suc_f:
cpv, file, comment = l.split("\t")
if comment.rstrip().lstrip() == "removed":
removed_files.setdefault(cpv,[]).append(file)
elif comment.lstrip().startswith("added"):
added_files.setdefault(cpv,[]).append(file)
suc_f.close()
if not os.path.isdir(sys.argv[5]):
sys.stderr.write("%s isn't a dir, thus isn't distdir. die.\n" % sys.argv[5])
sys.exit(6)
dist_files={}
for f in os.listdir(sys.argv[5]):
dist_files[f] = os.stat(sys.argv[5]+os.path.sep+f).st_size
del_list={}
del_totals={}
del_count=0
t = None
with io.open(scheduled_deletions_log, mode='r', encoding='utf_8') as f:
for line in f:
split = line.rstrip().split('\t')
if len(split) == 1:
try:
t = time.strftime("%s",
time.strptime(split[0], log_date_format))
except ValueError:
continue
# epoch time.
del_totals[t] = 0
l = {}
del_list[t] = l
elif len(split) == 3 and not split[0] and t is not None:
filename = split[1]
cpv = split[2]
l.setdefault(cpv, []).append(filename)
try:
del_totals[t] += dist_files[filename]
del_count += 1
except KeyError:
pass
cur_ep = time.strftime("%s", time.strptime(time.strftime("%Y-%m-%d",time.gmtime()), "%Y-%m-%d"))
l=list(del_list.keys())
l.sort()
while len(l) and cur_ep > l[0]:
del del_list[l[0]]
l.pop(0)
slated_for_death_total = 0
for x in l: slated_for_death_total += del_totals[x]
f_len=max(len(str(len(list(dist_files.keys())))), len(str(del_count)))
dist_sum=sum(dist_files.values())
d_len = max(len(str(dist_sum)), len(str(slated_for_death_total)))
outf.write("""<?xml version='1.0'?>
<?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl"?>
<guide link="failure.xml">
<title>Distfiles Mirroring Changes Report</title>
<version>1.0</version>
<date>"""+time.asctime(time.localtime())+"""</date>
<chapter><title>Master Stats</title><section><body>
<p>Report was generated on %s</p>
<impo>Remember the tree could've changed since then.</impo>
<p>%s bytes: %s files: Mirrored<br/>
%s bytes: %s files: Slated for deletion<br/>
Added %i files since beginning of day (UTC)<br/>
Removed %i files in last run</p></body></section></chapter>""" % \
(time.strftime("%A %b %d %Y, %H:%M:%S UTC", time.gmtime()), str(dist_sum).rjust(d_len), str(len(list(dist_files.keys()))).rjust(f_len),
str(slated_for_death_total).rjust(d_len), str(del_count).rjust(f_len),
sum([len(x) for x in list(added_files.values())]), sum([len(x) for x in list(removed_files.values())])))
dist_files.clear()
outf.write("""
<chapter><title>Failed Fetches</title>
<section><body>
<impo>Why has a file failed to be fetched?</impo>
<p>Upstream re-releases, a large failure in upstream mirrors/host, or flat out invalid URL's are typically the cause. Broken mirror
specifications also (mirror:/gentoo is not valid for example, must be mirror://gentoo). Odd ports for
the host can cause issues also (guidelines for ebuilds suggest standard ports for URI hosts).
</p><p>In other words, there <b>is</b> something wrong with the ebuild referenced, so please check the ebuild. It's
more likely then not broke in some fashion even if the maintainer isn't aware of it.</p>
<table><tr><th>Ebuild</th><th>File</th><th>Reason</th></tr>\n""")
#quote stripping is needed.
esort = list(failed_ebuilds.keys())
esort.sort()
for cpv in esort:
for f,r in failed_ebuilds[cpv]:
outf.write("<tr><ti>%s</ti><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(f), escape_xml(r)))
outf.write("</table>\n")
outf.write("""</body></section></chapter>
<chapter><title>Files Added</title><section><body><table><tr><th>Ebuild</th><th>File</th></tr>\n""")
added_s = list(added_files.keys())
added_s.sort()
for cpv in added_s:
added_files[cpv].sort()
for f in added_files[cpv]:
outf.write("<tr><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(f)))
outf.write("""</table></body></section></chapter>
<chapter><title>Files Removed</title><section><body><table><tr><th>Ebuild</th><th>File</th></tr>\n\n""")
rem_s = list(removed_files.keys())
rem_s.sort()
for cpv in rem_s:
removed_files[cpv].sort()
count = 0
for f in removed_files[cpv]:
if count == 0:
outf.write(""" <tr><ti>%s</ti>""" % escape_xml(cpv))
count+=1
else:
outf.write(""" <tr><ti></ti>""")
outf.write("""<ti>%s</ti></tr>\n""" % escape_xml(f))
outf.write("""</table></body></section></chapter>
<chapter><title>Scheduled Deletions</title><section><body>\n<ul>""")
l=list(del_list.keys())
l.sort()
for date in l:
outf.write(""" <li><uri link="#%s">%s</uri>, %i files for %i bytes</li>\n""" % (date, time.strftime("%B %d %Y",
time.gmtime(float(date))), sum([len(x) for x in list(del_list[date].values())]), del_totals[date]))
outf.write("</ul></body></section>\n")
for date in l:
outf.write("""<section id="%s"><title>Deletions for %s</title>
<body><table><tr><th>Ebuild (If known)</th><th>File</th></tr>\n""" % (date, time.strftime("%A %B %d %Y", time.gmtime(float(date)))))
cpvs = list(del_list[date].keys())
cpvs.sort()
for cpv in cpvs:
l=del_list[date][cpv]
l.sort()
outf.write(" <tr><ti>%s</ti><ti>%s</ti></tr>\n" % (escape_xml(cpv), escape_xml(l[0])))
l.pop(0)
for f in l:
outf.write(" <tr><ti></ti><ti>%s</ti></tr>\n" % escape_xml(f))
outf.write("</table>\n</body>\n</section>\n")
outf.write("</chapter>")
outf.write("<chapter><title>White Lists</title>")
if not whitelist:
outf.write("<section><body><p><uri link=\"whitelists.xml\">Whitelist Report</uri></p></body></section>")
#outf.write("<section><body><p>No whitelists.</p></body></section>")
l=list(whitelist.keys())
l.sort()
for x in l:
outf.write("<section><title>%s</title><body><ul>\n" % os.path.basename(x))
whitelist[x].sort()
for y in whitelist[x]:
outf.write(" <li>%s</li>\n" % escape_xml(y))
outf.write("</ul></body></section>\n")
outf.write("</chapter></guide>")
outf.close()
|
GitWeb