Subversion Repositories SvarDOS

Rev

Rev 803 | Rev 909 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
562 mateuszvis 1
<?php /*
2
 
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
734 bttr 6
  buildidx computes an index json file for the SvarDOS repository.
673 mateusz.vi 7
  it must be executed pointing to a directory that stores packages (*.svp)
562 mateuszvis 8
  files. buildidx will generate the index file and save it into the package
9
  repository.
10
 
11
  requires php-zip
12
 
775 mateusz.vi 13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
736 mateusz.vi 14
  16 feb 2022: added warning about overlong version strings and wild files location
719 mateusz.vi 15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
673 mateusz.vi 16
  14 feb 2022: packages are expected to have the *.svp extension
650 mateusz.vi 17
  12 feb 2022: skip source packages from being processed (*.src.zip)
562 mateuszvis 18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
19
  13 feb 2021: 'title' LSM field is no longer looked after
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
25
  07 dec 2013: rewritten buildidx in ANSI C89
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
27
  22 jul 2013: creating a listing.txt file with list of packages
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
31
  04 feb 2013: added CRC32 support
32
  22 sep 2012: forked 1st version from FDUPDATE builder
33
*/
34
 
791 mateusz.vi 35
$PVER = "20220217";
562 mateuszvis 36
 
37
 
38
// computes the BSD sum of a file and returns it
39
function file2bsum($fname) {
40
  $result = 0;
41
 
42
  $fd = fopen($fname, 'rb');
43
  if ($fd === false) return(0);
44
 
45
  while (!feof($fd)) {
46
 
47
    $buff = fread($fd, 1024 * 1024);
48
 
563 mateuszvis 49
    $slen = strlen($buff);
50
    for ($i = 0; $i < $slen; $i++) {
562 mateuszvis 51
      // rotr
52
      $result = ($result >> 1) | ($result << 15);
53
      // add and truncate to 16 bits
563 mateuszvis 54
      $result += ord($buff[$i]);
562 mateuszvis 55
      $result &= 0xffff;
56
    }
57
  }
58
 
59
  fclose($fd);
60
  return($result);
61
}
62
 
63
 
64
// reads file fil from zip archive z and returns its content, or false on error
65
function read_file_from_zip($z, $fil) {
66
  $zip = new ZipArchive;
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
68
    echo "ERROR: failed to open zip file '{$z}'\n";
69
    return(false);
70
  }
71
 
72
  // load the appinfo/pkgname.lsm file
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
74
 
75
  $zip->close();
76
  return($res);
77
}
78
 
79
 
731 mateusz.vi 80
function read_list_of_files_in_zip($z) {
81
  $zip = new ZipArchive;
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
83
    echo "ERROR: failed to open zip file '{$z}'\n";
84
    return(false);
85
  }
86
 
87
  $res = array();
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
89
 
90
  $zip->close();
91
  return($res);
92
}
93
 
94
 
562 mateuszvis 95
// reads a LSM string and returns it in the form of an array
96
function parse_lsm($s) {
97
  $res = array();
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
99
    // the line is "token: value", let's find the colon
100
    $colpos = strpos($l, ':');
101
    if (($colpos === false) || ($colpos === 0)) continue;
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
103
    $val = trim(substr($l, $colpos + 1));
104
    $res[$tok] = $val;
105
  }
106
  return($res);
107
}
108
 
109
 
731 mateusz.vi 110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
111
function str_head_is($haystack, $needle) {
112
  return strpos($haystack, $needle) === 0;
113
}
114
 
115
 
791 mateusz.vi 116
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
117
function load_core_list($repodir) {
118
  $res = array();
119
 
120
  foreach (scandir($repodir . '/core/') as $f) {
121
    if (!preg_match('/\.svp$/', $f)) continue;
122
    $res[] = explode('.', $f)[0];
123
  }
124
  return($res);
125
}
126
 
127
 
562 mateuszvis 128
// ***************** MAIN ROUTINE *********************************************
129
 
719 mateusz.vi 130
//echo "SvarDOS repository index generator ver {$PVER}\n";
562 mateuszvis 131
 
132
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
133
  echo "usage: php buildidx.php repodir\n";
134
  exit(1);
135
}
136
 
137
$repodir = $_SERVER['argv'][1];
138
 
139
$pkgfiles = scandir($repodir);
140
$pkgcount = 0;
141
 
738 mateusz.vi 142
 
795 mateusz.vi 143
// load the list of CORE and MSDOS_COMPAT packages
738 mateusz.vi 144
 
791 mateusz.vi 145
$core_packages_list = load_core_list($repodir);
804 bttr 146
$msdos_compat_list = explode(' ', 'append assign attrib chkdsk choice command comp cpidos debug defrag deltree diskcomp diskcopy display edit edlin exe2bin fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mirror mode more move nlsfunc print replace share shsucdx sort swsubst tree undelete unformat xcopy');
738 mateusz.vi 147
 
719 mateusz.vi 148
// do a list of all svp packages with their available versions and descriptions
562 mateuszvis 149
 
719 mateusz.vi 150
$pkgdb = array();
151
foreach ($pkgfiles as $fname) {
801 mateusz.vi 152
  if (!preg_match('/\.svp$/i', $fname)) continue; // skip non-svp files
562 mateuszvis 153
 
801 mateusz.vi 154
  if (!preg_match('/^[a-zA-Z0-9+. _-]*\.svp$/', $fname)) {
155
    echo "ERROR: {$fname} has a very weird name\n";
156
    continue;
157
  }
158
 
719 mateusz.vi 159
  $path_parts = pathinfo($fname);
160
  $pkgnam = explode('-', $path_parts['filename'])[0];
161
  $pkgfullpath = realpath($repodir . '/' . $fname);
562 mateuszvis 162
 
719 mateusz.vi 163
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
562 mateuszvis 164
  if ($lsm == false) {
802 mateusz.vi 165
    echo "ERROR: {$fname} does not contain an LSM file at the expected location\n";
719 mateusz.vi 166
    continue;
562 mateuszvis 167
  }
168
  $lsmarray = parse_lsm($lsm);
169
  if (empty($lsmarray['version'])) {
719 mateusz.vi 170
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
171
    continue;
562 mateuszvis 172
  }
730 mateusz.vi 173
  if (strlen($lsmarray['version']) > 16) {
737 mateusz.vi 174
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
730 mateusz.vi 175
    continue;
176
  }
562 mateuszvis 177
  if (empty($lsmarray['description'])) {
719 mateusz.vi 178
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
179
    continue;
562 mateuszvis 180
  }
181
 
731 mateusz.vi 182
  // validate the files present in the archive
183
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
739 mateusz.vi 184
  $pkgdir = $pkgnam;
185
 
768 mateusz.vi 186
  // special rule for "parent and children" packages
187
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
754 mateusz.vi 188
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
802 mateusz.vi 189
  if ($pkgnam == 'clamdb') $pkgdir = 'clamav'; // data patterns for clamav
739 mateusz.vi 190
 
768 mateusz.vi 191
  // array used to detect duplicated entries after lower-case conversion
192
  $duparr = array();
193
 
731 mateusz.vi 194
  foreach ($listoffiles as $f) {
195
    $f = strtolower($f);
768 mateusz.vi 196
    $path_array = explode('/', $f);
197
    // emit a warning when non-8+3 filenames are spotted and find duplicates
198
    foreach ($path_array as $item) {
199
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
200
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
201
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
202
      }
203
    }
204
    // look for dups
205
    if (array_search($f, $duparr) !== false) {
206
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
207
    } else {
208
      $duparr[] = $f;
209
    }
731 mateusz.vi 210
    // LSM file is ok
211
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
212
    if ($f === "appinfo/") continue;
795 mateusz.vi 213
    // CORE and MSDOS_COMPAT packages are premium citizens and can do a little more
214
    if ((array_search($pkgnam, $core_packages_list) !== false)
215
       || (array_search($pkgnam, $msdos_compat_list) !== false)) {
736 mateusz.vi 216
      if (str_head_is($f, 'bin/')) continue;
779 mateusz.vi 217
      if (str_head_is($f, 'cpi/')) continue;
749 mateusz.vi 218
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
219
      if ($f === 'doc/') continue;
220
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
221
      if ($f === 'nls/') continue;
736 mateusz.vi 222
    }
798 mateusz.vi 223
    // the help package is allowed to put files in... help
224
    if (($pkgnam == 'help') && (str_head_is($f, 'help/'))) continue;
749 mateusz.vi 225
    // well-known "category" dirs are okay
739 mateusz.vi 226
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
731 mateusz.vi 227
    if ($f === 'progs/') continue;
739 mateusz.vi 228
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
731 mateusz.vi 229
    if ($f === 'devel/') continue;
739 mateusz.vi 230
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
731 mateusz.vi 231
    if ($f === 'games/') continue;
739 mateusz.vi 232
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
731 mateusz.vi 233
    if ($f === 'drivers/') continue;
768 mateusz.vi 234
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
731 mateusz.vi 235
  }
236
 
719 mateusz.vi 237
  $meta['fname'] = $fname;
238
  $meta['desc'] = $lsmarray['description'];
239
 
240
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
241
}
242
 
801 mateusz.vi 243
 
719 mateusz.vi 244
$db = array();
245
 
246
// iterate over each svp package
247
foreach ($pkgdb as $pkg => $versions) {
248
 
249
  // sort filenames by version, highest first
250
  uksort($versions, "version_compare");
251
  $versions = array_reverse($versions, true);
252
 
253
  foreach ($versions as $ver => $meta) {
254
    $fname = $meta['fname'];
255
    $desc = $meta['desc'];
256
 
257
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
258
 
259
    $meta2['ver'] = strval($ver);
260
    $meta2['bsum'] = $bsum;
261
 
262
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
263
    $db[$pkg]['versions'][$fname] = $meta2;
264
  }
265
 
562 mateuszvis 266
  $pkgcount++;
267
 
268
}
269
 
719 mateusz.vi 270
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
562 mateuszvis 271
 
801 mateusz.vi 272
$json_blob = json_encode($db);
273
if ($json_blob === false) {
274
  echo "ERROR: JSON convertion failed! -> ";
275
  switch (json_last_error()) {
276
    case JSON_ERROR_DEPTH:
277
      echo 'maximum stack depth exceeded';
278
      break;
279
    case JSON_ERROR_STATE_MISMATCH:
280
      echo 'underflow of the modes mismatch';
281
      break;
282
    case JSON_ERROR_CTRL_CHAR:
283
      echo 'unexpected control character found';
284
      break;
285
    case JSON_ERROR_UTF8:
286
      echo 'malformed utf-8 characters';
287
      break;
288
    default:
289
      echo "unknown error";
290
      break;
291
  }
292
  echo "\n";
293
}
294
 
719 mateusz.vi 295
file_put_contents($repodir . '/_index.json', json_encode($db));
562 mateuszvis 296
 
297
exit(0);
298
 
299
?>