Subversion Repositories SvarDOS

Rev

Rev 798 | Rev 802 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
562 mateuszvis 1
<?php /*
2
 
3
  SvarDOS repo index builder
4
  Copyright (C) Mateusz Viste 2012-2022
5
 
734 bttr 6
  buildidx computes an index json file for the SvarDOS repository.
673 mateusz.vi 7
  it must be executed pointing to a directory that stores packages (*.svp)
562 mateuszvis 8
  files. buildidx will generate the index file and save it into the package
9
  repository.
10
 
11
  requires php-zip
12
 
775 mateusz.vi 13
  17 feb 2022: checking for non-8+3 filenames in packages and duplicates + devload no longer part of CORE
736 mateusz.vi 14
  16 feb 2022: added warning about overlong version strings and wild files location
719 mateusz.vi 15
  15 feb 2022: index is generated as json, contains all filenames and alt versions
673 mateusz.vi 16
  14 feb 2022: packages are expected to have the *.svp extension
650 mateusz.vi 17
  12 feb 2022: skip source packages from being processed (*.src.zip)
562 mateuszvis 18
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
19
  13 feb 2021: 'title' LSM field is no longer looked after
20
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
21
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
22
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
23
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
24
  27 aug 2016: accepting full paths to repos (starting with /...)
25
  07 dec 2013: rewritten buildidx in ANSI C89
26
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
27
  22 jul 2013: creating a listing.txt file with list of packages
28
  18 jul 2013: writing the number of packaged into the first line of the lst file
29
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
30
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
31
  04 feb 2013: added CRC32 support
32
  22 sep 2012: forked 1st version from FDUPDATE builder
33
*/
34
 
791 mateusz.vi 35
$PVER = "20220217";
562 mateuszvis 36
 
37
 
38
// computes the BSD sum of a file and returns it
39
function file2bsum($fname) {
40
  $result = 0;
41
 
42
  $fd = fopen($fname, 'rb');
43
  if ($fd === false) return(0);
44
 
45
  while (!feof($fd)) {
46
 
47
    $buff = fread($fd, 1024 * 1024);
48
 
563 mateuszvis 49
    $slen = strlen($buff);
50
    for ($i = 0; $i < $slen; $i++) {
562 mateuszvis 51
      // rotr
52
      $result = ($result >> 1) | ($result << 15);
53
      // add and truncate to 16 bits
563 mateuszvis 54
      $result += ord($buff[$i]);
562 mateuszvis 55
      $result &= 0xffff;
56
    }
57
  }
58
 
59
  fclose($fd);
60
  return($result);
61
}
62
 
63
 
64
// reads file fil from zip archive z and returns its content, or false on error
65
function read_file_from_zip($z, $fil) {
66
  $zip = new ZipArchive;
67
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
68
    echo "ERROR: failed to open zip file '{$z}'\n";
69
    return(false);
70
  }
71
 
72
  // load the appinfo/pkgname.lsm file
73
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
74
 
75
  $zip->close();
76
  return($res);
77
}
78
 
79
 
731 mateusz.vi 80
function read_list_of_files_in_zip($z) {
81
  $zip = new ZipArchive;
82
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
83
    echo "ERROR: failed to open zip file '{$z}'\n";
84
    return(false);
85
  }
86
 
87
  $res = array();
88
  for ($i = 0; $i < $zip->numFiles; $i++) $res[] = $zip->getNameIndex($i);
89
 
90
  $zip->close();
91
  return($res);
92
}
93
 
94
 
562 mateuszvis 95
// reads a LSM string and returns it in the form of an array
96
function parse_lsm($s) {
97
  $res = array();
98
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
99
    // the line is "token: value", let's find the colon
100
    $colpos = strpos($l, ':');
101
    if (($colpos === false) || ($colpos === 0)) continue;
102
    $tok = strtolower(trim(substr($l, 0, $colpos)));
103
    $val = trim(substr($l, $colpos + 1));
104
    $res[$tok] = $val;
105
  }
106
  return($res);
107
}
108
 
109
 
731 mateusz.vi 110
// on PHP 8+ there is str_starts_with(), but not on PHP 7 so I use this
111
function str_head_is($haystack, $needle) {
112
  return strpos($haystack, $needle) === 0;
113
}
114
 
115
 
791 mateusz.vi 116
// returns an array that contains CORE packages (populated from the core subdirectory in pkgdir)
117
function load_core_list($repodir) {
118
  $res = array();
119
 
120
  foreach (scandir($repodir . '/core/') as $f) {
121
    if (!preg_match('/\.svp$/', $f)) continue;
122
    $res[] = explode('.', $f)[0];
123
  }
124
  return($res);
125
}
126
 
127
 
562 mateuszvis 128
// ***************** MAIN ROUTINE *********************************************
129
 
719 mateusz.vi 130
//echo "SvarDOS repository index generator ver {$PVER}\n";
562 mateuszvis 131
 
132
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
133
  echo "usage: php buildidx.php repodir\n";
134
  exit(1);
135
}
136
 
137
$repodir = $_SERVER['argv'][1];
138
 
139
$pkgfiles = scandir($repodir);
140
$pkgcount = 0;
141
 
738 mateusz.vi 142
 
795 mateusz.vi 143
// load the list of CORE and MSDOS_COMPAT packages
738 mateusz.vi 144
 
791 mateusz.vi 145
$core_packages_list = load_core_list($repodir);
797 mateusz.vi 146
$msdos_compat_list = explode(' ', 'append assign attrib chkdsk choice command cpidos debug defrag deltree diskcopy display edit edlin fc fdapm fdisk find format help himemx kernel keyb label localcfg mem mode more move nlsfunc print replace share shsucdx sort swsubst tree undelete unformat xcopy');
738 mateusz.vi 147
 
719 mateusz.vi 148
// do a list of all svp packages with their available versions and descriptions
562 mateuszvis 149
 
719 mateusz.vi 150
$pkgdb = array();
151
foreach ($pkgfiles as $fname) {
801 mateusz.vi 152
  if (!preg_match('/\.svp$/i', $fname)) continue; // skip non-svp files
562 mateuszvis 153
 
801 mateusz.vi 154
  if (!preg_match('/^[a-zA-Z0-9+. _-]*\.svp$/', $fname)) {
155
    echo "ERROR: {$fname} has a very weird name\n";
156
    continue;
157
  }
158
 
719 mateusz.vi 159
  $path_parts = pathinfo($fname);
160
  $pkgnam = explode('-', $path_parts['filename'])[0];
161
  $pkgfullpath = realpath($repodir . '/' . $fname);
562 mateuszvis 162
 
719 mateusz.vi 163
  $lsm = read_file_from_zip($pkgfullpath, "appinfo/{$pkgnam}.lsm");
562 mateuszvis 164
  if ($lsm == false) {
719 mateusz.vi 165
    echo "ERROR: pkg {$fname} does not contain an LSM file at the expected location\n";
166
    continue;
562 mateuszvis 167
  }
168
  $lsmarray = parse_lsm($lsm);
169
  if (empty($lsmarray['version'])) {
719 mateusz.vi 170
    echo "ERROR: lsm file in {$fname} does not contain a version\n";
171
    continue;
562 mateuszvis 172
  }
730 mateusz.vi 173
  if (strlen($lsmarray['version']) > 16) {
737 mateusz.vi 174
    echo "ERROR: version string in lsm file of {$fname} is too long (16 chars max)\n";
730 mateusz.vi 175
    continue;
176
  }
562 mateuszvis 177
  if (empty($lsmarray['description'])) {
719 mateusz.vi 178
    echo "ERROR: lsm file in {$fname} does not contain a description\n";
179
    continue;
562 mateuszvis 180
  }
181
 
731 mateusz.vi 182
  // validate the files present in the archive
183
  $listoffiles = read_list_of_files_in_zip($pkgfullpath);
739 mateusz.vi 184
  $pkgdir = $pkgnam;
185
 
768 mateusz.vi 186
  // special rule for "parent and children" packages
187
  if (str_head_is($pkgnam, 'djgpp_')) $pkgdir = 'djgpp'; // djgpp_* packages put their files in djgpp
754 mateusz.vi 188
  if ($pkgnam == 'fbc_help') $pkgdir = 'fbc'; // FreeBASIC help goes to the FreeBASIC dir
739 mateusz.vi 189
 
768 mateusz.vi 190
  // array used to detect duplicated entries after lower-case conversion
191
  $duparr = array();
192
 
731 mateusz.vi 193
  foreach ($listoffiles as $f) {
194
    $f = strtolower($f);
768 mateusz.vi 195
    $path_array = explode('/', $f);
196
    // emit a warning when non-8+3 filenames are spotted and find duplicates
197
    foreach ($path_array as $item) {
198
      if (empty($item)) continue; // skip empty items at end of paths (eg. appinfo/)
199
      if (!preg_match("/[a-z0-9!#$%&'()@^_`{}~-]{1,8}(\.[a-z0-9!#$%&'()@^_`{}~-]{1,3}){0,1}/", $item)) {
200
        echo "WARNING: {$fname} contains a non-8+3 path (or weird char): {$item} (in $f)\n";
201
      }
202
    }
203
    // look for dups
204
    if (array_search($f, $duparr) !== false) {
205
      echo "WARNING: {$fname} contains a duplicated entry: '{$f}'\n";
206
    } else {
207
      $duparr[] = $f;
208
    }
731 mateusz.vi 209
    // LSM file is ok
210
    if ($f === "appinfo/{$pkgnam}.lsm") continue;
211
    if ($f === "appinfo/") continue;
795 mateusz.vi 212
    // CORE and MSDOS_COMPAT packages are premium citizens and can do a little more
213
    if ((array_search($pkgnam, $core_packages_list) !== false)
214
       || (array_search($pkgnam, $msdos_compat_list) !== false)) {
736 mateusz.vi 215
      if (str_head_is($f, 'bin/')) continue;
779 mateusz.vi 216
      if (str_head_is($f, 'cpi/')) continue;
749 mateusz.vi 217
      if (str_head_is($f, "doc/{$pkgdir}/")) continue;
218
      if ($f === 'doc/') continue;
219
      if (str_head_is($f, "nls/{$pkgdir}.")) continue;
220
      if ($f === 'nls/') continue;
736 mateusz.vi 221
    }
798 mateusz.vi 222
    // the help package is allowed to put files in... help
223
    if (($pkgnam == 'help') && (str_head_is($f, 'help/'))) continue;
749 mateusz.vi 224
    // well-known "category" dirs are okay
739 mateusz.vi 225
    if (str_head_is($f, "progs/{$pkgdir}/")) continue;
731 mateusz.vi 226
    if ($f === 'progs/') continue;
739 mateusz.vi 227
    if (str_head_is($f, "devel/{$pkgdir}/")) continue;
731 mateusz.vi 228
    if ($f === 'devel/') continue;
739 mateusz.vi 229
    if (str_head_is($f, "games/{$pkgdir}/")) continue;
731 mateusz.vi 230
    if ($f === 'games/') continue;
739 mateusz.vi 231
    if (str_head_is($f, "drivers/{$pkgdir}/")) continue;
731 mateusz.vi 232
    if ($f === 'drivers/') continue;
768 mateusz.vi 233
    echo "WARNING: {$fname} contains a file in an illegal location: {$f}\n";
731 mateusz.vi 234
  }
235
 
719 mateusz.vi 236
  $meta['fname'] = $fname;
237
  $meta['desc'] = $lsmarray['description'];
238
 
239
  $pkgdb[$pkgnam][$lsmarray['version']] = $meta;
240
}
241
 
801 mateusz.vi 242
 
719 mateusz.vi 243
$db = array();
244
 
245
// iterate over each svp package
246
foreach ($pkgdb as $pkg => $versions) {
247
 
248
  // sort filenames by version, highest first
249
  uksort($versions, "version_compare");
250
  $versions = array_reverse($versions, true);
251
 
252
  foreach ($versions as $ver => $meta) {
253
    $fname = $meta['fname'];
254
    $desc = $meta['desc'];
255
 
256
    $bsum = file2bsum(realpath($repodir . '/' . $fname));
257
 
258
    $meta2['ver'] = strval($ver);
259
    $meta2['bsum'] = $bsum;
260
 
261
    if (empty($db[$pkg]['desc'])) $db[$pkg]['desc'] = $desc;
262
    $db[$pkg]['versions'][$fname] = $meta2;
263
  }
264
 
562 mateuszvis 265
  $pkgcount++;
266
 
267
}
268
 
719 mateusz.vi 269
if ($pkgcount < 100) echo "WARNING: an unexpectedly low number of packages has been found in the repo ({$pkgcount})\n";
562 mateuszvis 270
 
801 mateusz.vi 271
$json_blob = json_encode($db);
272
if ($json_blob === false) {
273
  echo "ERROR: JSON convertion failed! -> ";
274
  switch (json_last_error()) {
275
    case JSON_ERROR_DEPTH:
276
      echo 'maximum stack depth exceeded';
277
      break;
278
    case JSON_ERROR_STATE_MISMATCH:
279
      echo 'underflow of the modes mismatch';
280
      break;
281
    case JSON_ERROR_CTRL_CHAR:
282
      echo 'unexpected control character found';
283
      break;
284
    case JSON_ERROR_UTF8:
285
      echo 'malformed utf-8 characters';
286
      break;
287
    default:
288
      echo "unknown error";
289
      break;
290
  }
291
  echo "\n";
292
}
293
 
719 mateusz.vi 294
file_put_contents($repodir . '/_index.json', json_encode($db));
562 mateuszvis 295
 
296
exit(0);
297
 
298
?>