Subversion Repositories SvarDOS

Rev

Rev 781 | Rev 795 | Go to most recent revision | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 781 Rev 791
1
<?php /*
1
<?php /*
2
 
2
 
3
  SvarDOS repo index builder
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
5
 
6
  buildidx computes an index json file for the SvarDOS repository.
6
  buildidx computes an index json file for the SvarDOS repository.
7
  it must be executed pointing to a directory that stores packages (*.svp)
7
  it must be executed pointing to a directory that stores packages (*.svp)
8
  files. buildidx will generate the index file and save it into the package
8
  files. buildidx will generate the index file and save it into the package
9
  repository.
9
  repository.
10
 
10
 
11
  requires php-zip
11
  requires php-zip
12
 
12
 
13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
14
  16 feb 2022: added warning about overlong version strings and wild files location
14
  16 feb 2022: added warning about overlong version strings and wild files location
15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
16
  14 feb 2022: packages are expected to have the *.svp extension
16
  14 feb 2022: packages are expected to have the *.svp extension
17
  12 feb 2022: skip source packages from being processed (*.src.zip)
17
  12 feb 2022: skip source packages from being processed (*.src.zip)
18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
19
  13 feb 2021: 'title' LSM field is no longer looked after
19
  13 feb 2021: 'title' LSM field is no longer looked after
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
25
  07 dec 2013: rewritten buildidx in ANSI C89
25
  07 dec 2013: rewritten buildidx in ANSI C89
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
27
  22 jul 2013: creating a listing.txt file with list of packages
27
  22 jul 2013: creating a listing.txt file with list of packages
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
31
  04 feb 2013: added CRC32 support
31
  04 feb 2013: added CRC32 support
32
  22 sep 2012: forked 1st version from FDUPDATE builder
32
  22 sep 2012: forked 1st version from FDUPDATE builder
33
*/
33
*/
34
 
34
 
35
$PVER = "20220216";
35
$PVER = "20220217";
36
 
36
 
37
 
37
 
38
// computes the BSD sum of a file and returns it
38
// computes the BSD sum of a file and returns it
39
function file2bsum($fname) {
39
function file2bsum($fname) {
40
  $result = 0;
40
  $result = 0;
41
 
41
 
42
  $fd = fopen($fname, 'rb');
42
  $fd = fopen($fname, 'rb');
43
  if ($fd === false) return(0);
43
  if ($fd === false) return(0);
44
 
44
 
45
  while (!feof($fd)) {
45
  while (!feof($fd)) {
46
 
46
 
47
    $buff = fread($fd, 1024 * 1024);
47
    $buff = fread($fd, 1024 * 1024);
48
 
48
 
49
    $slen = strlen($buff);
49
    $slen = strlen($buff);
50
    for ($i = 0; $i < $slen; $i++) {
50
    for ($i = 0; $i < $slen; $i++) {
51
      // rotr
51
      // rotr
52
      $result = ($result >> 1) | ($result << 15);
52
      $result = ($result >> 1) | ($result << 15);
53
      // add and truncate to 16 bits
53
      // add and truncate to 16 bits
54
      $result += ord($buff[$i]);
54
      $result += ord($buff[$i]);
55
      $result &= 0xffff;
55
      $result &= 0xffff;
56
    }
56
    }
57
  }
57
  }
58
 
58
 
59
  fclose($fd);
59
  fclose($fd);
60
  return($result);
60
  return($result);
61
}
61
}
62
 
62
 
63
 
63
 
64
// reads file fil from zip archive z and returns its content, or false on error
64
// reads file fil from zip archive z and returns its content, or false on error
65
function read_file_from_zip($z, $fil) {
65
function read_file_from_zip($z, $fil) {
66
  $zip = new ZipArchive;
66
  $zip = new ZipArchive;
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
68
    echo "ERROR: failed to open zip file '{$z}'\n";
68
    echo "ERROR: failed to open zip file '{$z}'\n";
69
    return(false);
69
    return(false);
70
  }
70
  }
71
 
71
 
72
  // load the appinfo/pkgname.lsm file
72
  // load the appinfo/pkgname.lsm file
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
74
 
74
 
75
  $zip->close();
75
  $zip->close();
76
  return($res);
76
  return($res);
77
}
77
}
78
 
78
 
79
 
79
 
80
function read_list_of_files_in_zip($z) {
80
function read_list_of_files_in_zip($z) {
81
  $zip = new ZipArchive;
81
  $zip = new ZipArchive;
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
83
    echo "ERROR: failed to open zip file '{$z}'\n";
83
    echo "ERROR: failed to open zip file '{$z}'\n";
84
    return(false);
84
    return(false);
85
  }
85
  }
86
 
86
 
87
  $res = array();
87
  $res = array();
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
89
 
89
 
90
  $zip->close();
90
  $zip->close();
91
  return($res);
91
  return($res);
92
}
92
}
93
 
93
 
94
 
94
 
95
// reads a LSM string and returns it in the form of an array
95
// reads a LSM string and returns it in the form of an array
96
function parse_lsm($s) {
96
function parse_lsm($s) {
97
  $res = array();
97
  $res = array();
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
99
    // the line is "token: value", let's find the colon
99
    // the line is "token: value", let's find the colon
100
    $colpos = strpos($l, ':');
100
    $colpos = strpos($l, ':');
101
    if (($colpos === false) || ($colpos === 0)) continue;
101
    if (($colpos === false) || ($colpos === 0)) continue;
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
103
    $val = trim(substr($l, $colpos + 1));
103
    $val = trim(substr($l, $colpos + 1));
104
    $res[$tok] = $val;
104
    $res[$tok] = $val;
105
  }
105
  }
106
  return($res);
106
  return($res);
107
}
107
}
108
 
108
 
109
 
109
 
110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
111
function str_head_is($haystack, $needle) {
111
function str_head_is($haystack, $needle) {
112
  return strpos($haystack, $needle) === 0;
112
  return strpos($haystack, $needle) === 0;
113
}
113
}
114
 
114
 
115
 
115
 
-
 
116
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
-
 
117
function load_core_list($repodir) {
-
 
118
  $res = array();
-
 
119
 
-
 
120
  foreach (scandir($repodir . '/core/') as $f) {
-
 
121
    if (!preg_match('/\.svp$/', $f)) continue;
-
 
122
    $res[] = explode('.', $f)[0];
-
 
123
  }
-
 
124
  return($res);
-
 
125
}
-
 
126
 
-
 
127
 
116
// ***************** MAIN ROUTINE *********************************************
128
// ***************** MAIN ROUTINE *********************************************
117
 
129
 
118
//echo "SvarDOS repository index generator ver {$PVER}\n";
130
//echo "SvarDOS repository index generator ver {$PVER}\n";
119
 
131
 
120
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
132
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
121
  echo "usage: php buildidx.php repodir\n";
133
  echo "usage: php buildidx.php repodir\n";
122
  exit(1);
134
  exit(1);
123
}
135
}
124
 
136
 
125
$repodir = $_SERVER['argv'][1];
137
$repodir = $_SERVER['argv'][1];
126
 
138
 
127
$pkgfiles = scandir($repodir);
139
$pkgfiles = scandir($repodir);
128
$pkgcount = 0;
140
$pkgcount = 0;
129
 
141
 
130
 
142
 
131
// load the list of CORE packages
143
// load the list of CORE packages
132
 
144
 
133
$core_packages_list = explode(' ', 'amb attrib chkdsk choice command cpidos debug deltree diskcopy display dosfsck edit fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mode more move pkg pkgnet shsucdx sort tree');
145
$core_packages_list = load_core_list($repodir);
134
 
146
 
135
 
147
 
136
// do a list of all svp packages with their available versions and descriptions
148
// do a list of all svp packages with their available versions and descriptions
137
 
149
 
138
$pkgdb = array();
150
$pkgdb = array();
139
foreach ($pkgfiles as $fname) {
151
foreach ($pkgfiles as $fname) {
140
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
152
  if (!preg_match('/.svp$/i', $fname)) continue; // skip non-svp files
141
 
153
 
142
  $path_parts = pathinfo($fname);
154
  $path_parts = pathinfo($fname);
143
  $pkgnam = explode('-', $path_parts['filename'])[0];
155
  $pkgnam = explode('-', $path_parts['filename'])[0];
144
  $pkgfullpath = realpath($repodir . '/' . $fname);
156
  $pkgfullpath = realpath($repodir . '/' . $fname);
145
 
157
 
146
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
158
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
147
  if ($lsm == false) {
159
  if ($lsm == false) {
148
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
160
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
149
    continue;
161
    continue;
150
  }
162
  }
151
  $lsmarray = parse_lsm($lsm);
163
  $lsmarray = parse_lsm($lsm);
152
  if (empty($lsmarray['version'])) {
164
  if (empty($lsmarray['version'])) {
153
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
165
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
154
    continue;
166
    continue;
155
  }
167
  }
156
  if (strlen($lsmarray['version']) > 16) {
168
  if (strlen($lsmarray['version']) > 16) {
157
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
169
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
158
    continue;
170
    continue;
159
  }
171
  }
160
  if (empty($lsmarray['description'])) {
172
  if (empty($lsmarray['description'])) {
161
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
173
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
162
    continue;
174
    continue;
163
  }
175
  }
164
 
176
 
165
  // validate the files present in the archive
177
  // validate the files present in the archive
166
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
178
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
167
  $pkgdir = $pkgnam;
179
  $pkgdir = $pkgnam;
168
 
180
 
169
  // special rule for "parent and children" packages
181
  // special rule for "parent and children" packages
170
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
182
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
171
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
183
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
172
 
184
 
173
  // array used to detect duplicated entries after lower-case conversion
185
  // array used to detect duplicated entries after lower-case conversion
174
  $duparr = array();
186
  $duparr = array();
175
 
187
 
176
  foreach ($listoffiles as $f) {
188
  foreach ($listoffiles as $f) {
177
    $f = strtolower($f);
189
    $f = strtolower($f);
178
    $path_array = explode('/', $f);
190
    $path_array = explode('/', $f);
179
    // emit a warning when non-8+3 filenames are spotted and find duplicates
191
    // emit a warning when non-8+3 filenames are spotted and find duplicates
180
    foreach ($path_array as $item) {
192
    foreach ($path_array as $item) {
181
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
193
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
182
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
194
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
183
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
195
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
184
      }
196
      }
185
    }
197
    }
186
    // look for dups
198
    // look for dups
187
    if (array_search($f, $duparr) !== false) {
199
    if (array_search($f, $duparr) !== false) {
188
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
200
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
189
    } else {
201
    } else {
190
      $duparr[] = $f;
202
      $duparr[] = $f;
191
    }
203
    }
192
    // LSM file is ok
204
    // LSM file is ok
193
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
205
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
194
    if ($f === "appinfo/") continue;
206
    if ($f === "appinfo/") continue;
195
    // CORE packages are premium citizens and can do a little more
207
    // CORE packages are premium citizens and can do a little more
196
    if (array_search($pkgnam, $core_packages_list) !== false) {
208
    if (array_search($pkgnam, $core_packages_list) !== false) {
197
      if (str_head_is($f, 'bin/')) continue;
209
      if (str_head_is($f, 'bin/')) continue;
198
      if (str_head_is($f, 'cpi/')) continue;
210
      if (str_head_is($f, 'cpi/')) continue;
199
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
211
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
200
      if ($f === 'doc/') continue;
212
      if ($f === 'doc/') continue;
201
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
213
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
202
      if ($f === 'nls/') continue;
214
      if ($f === 'nls/') continue;
203
    }
215
    }
204
    // well-known "category" dirs are okay
216
    // well-known "category" dirs are okay
205
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
217
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
206
    if ($f === 'progs/') continue;
218
    if ($f === 'progs/') continue;
207
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
219
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
208
    if ($f === 'devel/') continue;
220
    if ($f === 'devel/') continue;
209
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
221
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
210
    if ($f === 'games/') continue;
222
    if ($f === 'games/') continue;
211
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
223
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
212
    if ($f === 'drivers/') continue;
224
    if ($f === 'drivers/') continue;
213
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
225
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
214
  }
226
  }
215
 
227
 
216
  $meta['fname'] = $fname;
228
  $meta['fname'] = $fname;
217
  $meta['desc'] = $lsmarray['description'];
229
  $meta['desc'] = $lsmarray['description'];
218
 
230
 
219
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
231
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
220
}
232
}
221
 
233
 
222
$db = array();
234
$db = array();
223
 
235
 
224
// iterate over each svp package
236
// iterate over each svp package
225
foreach ($pkgdb as $pkg => $versions) {
237
foreach ($pkgdb as $pkg => $versions) {
226
 
238
 
227
  // sort filenames by version, highest first
239
  // sort filenames by version, highest first
228
  uksort($versions, "version_compare");
240
  uksort($versions, "version_compare");
229
  $versions = array_reverse($versions, true);
241
  $versions = array_reverse($versions, true);
230
 
242
 
231
  foreach ($versions as $ver => $meta) {
243
  foreach ($versions as $ver => $meta) {
232
    $fname = $meta['fname'];
244
    $fname = $meta['fname'];
233
    $desc = $meta['desc'];
245
    $desc = $meta['desc'];
234
 
246
 
235
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
247
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
236
 
248
 
237
    $meta2['ver'] = strval($ver);
249
    $meta2['ver'] = strval($ver);
238
    $meta2['bsum'] = $bsum;
250
    $meta2['bsum'] = $bsum;
239
 
251
 
240
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
252
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
241
    $db[$pkg]['versions'][$fname] = $meta2;
253
    $db[$pkg]['versions'][$fname] = $meta2;
242
  }
254
  }
243
 
255
 
244
  $pkgcount++;
256
  $pkgcount++;
245
 
257
 
246
}
258
}
247
 
259
 
248
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
260
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
249
 
261
 
250
file_put_contents($repodir . '/_index.json', json_encode($db));
262
file_put_contents($repodir . '/_index.json', json_encode($db));
251
 
263
 
252
exit(0);
264
exit(0);
253
 
265
 
254
?>
266
?>
255
 
267