WebSVN – SvarDOS – Diff – /buildidx/buildidx.php


<?php /*
 
  SvarDOS repo index builder
  Copyright (C) Mateusz Viste 2012-2022
 
  buildidx computes an index tsv file for the SvarDOS repository.
  it must be executed pointing to a directory that stores packages (*.svp)
  files. buildidx will generate the index file and save it into the package
  repository.
 
  requires php-zip
 
  14 feb 2022: packages are expected to have the *.svp extension
  12 feb 2022: skip source packages from being processed (*.src.zip)
  20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance
  13 feb 2021: 'title' LSM field is no longer looked after
  11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS
  13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index
  23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present
  28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)
  27 aug 2016: accepting full paths to repos (starting with /...)
  07 dec 2013: rewritten buildidx in ANSI C89
  19 aug 2013: add a compressed version of the index file to repos (index.gz)
  22 jul 2013: creating a listing.txt file with list of packages
  18 jul 2013: writing the number of packaged into the first line of the lst file
  11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files
  10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)
  04 feb 2013: added CRC32 support
  22 sep 2012: forked 1st version from FDUPDATE builder
*/
 
$PVER = "20220214";
 
 
// computes the BSD sum of a file and returns it
function file2bsum($fname) {
  $result = 0;
 
  $fd = fopen($fname, 'rb');
  if ($fd === false) return(0);
 
  while (!feof($fd)) {
 
    $buff = fread($fd, 1024 * 1024);
 
    $slen = strlen($buff);
    for ($i = 0; $i < $slen; $i++) {
      // rotr
      $result = ($result >> 1) | ($result << 15);
      // add and truncate to 16 bits
      $result += ord($buff[$i]);
      $result &= 0xffff;
    }
  }
 
  fclose($fd);
  return($result);
}
 
 
// reads file fil from zip archive z and returns its content, or false on error
function read_file_from_zip($z, $fil) {
  $zip = new ZipArchive;
  if ($zip->open($z, ZipArchive::RDONLY) !== true) {
    echo "ERROR: failed to open zip file '{$z}'\n";
    return(false);
  }
 
  // load the appinfo/pkgname.lsm file
  $res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);
 
  $zip->close();
  return($res);
}
 
 
// reads a LSM string and returns it in the form of an array
function parse_lsm($s) {
  $res = array();
  for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {
    // the line is "token: value", let's find the colon
    $colpos = strpos($l, ':');
    if (($colpos === false) || ($colpos === 0)) continue;
    $tok = strtolower(trim(substr($l, 0, $colpos)));
    $val = trim(substr($l, $colpos + 1));
    $res[$tok] = $val;
  }
  return($res);
}
 
 
// ***************** MAIN ROUTINE *********************************************
 
echo "SvarDOS repository index generator ver {$PVER}\n";
 
if (($_SERVER['argc'] != 2) || ($_SERVER['argv'][1][0] == '-')) {
  echo "usage: php buildidx.php repodir\n";
  exit(1);
}
 
$repodir = $_SERVER['argv'][1];
 
echo "building index for directory {$repodir}...\n";
 
$pkgfiles = scandir($repodir);
$pkglist = '';
$pkgcount = 0;
 
// iterate over each svp package
foreach ($pkgfiles as $zipfile) {
  if (!preg_match('/.svp$/i', $zipfile)) continue; // skip non-svp files
  if (strchr($zipfile, '-')) {
    echo "skipping: {$zipfile}\n";
    continue; // skip alt vers (like dosmid-0.9.2.svp)
  }
 
 
 
 
 
 
  $path_parts = pathinfo($zipfile);
  $pkg = strtolower($path_parts['filename']);
 
  $zipfile_fullpath = realpath($repodir . '/' . $zipfile);
 
  $lsm = read_file_from_zip($zipfile_fullpath, "appinfo/{$pkg}.lsm");
  if ($lsm == false) {
    echo "ERROR: pkg {$z} does not contain an LSM file at the expected location\n";
    exit(1);
  }
 
  $lsmarray = parse_lsm($lsm);
  if (empty($lsmarray['version'])) {
    echo "ERROR: lsm file in {$zipfile} does not contain a version\n";
    var_dump($lsmarray);
    exit(1);
  }
  if (empty($lsmarray['description'])) {
    echo "ERROR: lsm file in {$zipfile} does not contain a description\n";
    exit(1);
  }
 
  $pkglist .= "{$pkg}\t{$lsmarray['version']}\t{$lsmarray['description']}\t" . file2bsum($zipfile_fullpath) . "\n";
  $pkgcount++;
 
}
 
echo "DONE - processed " . $pkgcount . " svp packages\n";
 
file_put_contents($repodir . '/index.tsv', $pkglist);
 
exit(0);
 
?>
 

Subversion Repositories SvarDOS

(root)/buildidx/buildidx.php @ 734 – Rev 650 → 673

Rev 650		Rev 673
1	`<?php /*`	1	`<?php /*`
2		2
3	`SvarDOS repo index builder`	3	`SvarDOS repo index builder`
4	`Copyright (C) Mateusz Viste 2012-2022`	4	`Copyright (C) Mateusz Viste 2012-2022`
5		5
6	`buildidx computes an index tsv file for the SvarDOS repository.`	6	`buildidx computes an index tsv file for the SvarDOS repository.`
7	`it must be executed pointing to a directory that stores packages (zip)`	7	`it must be executed pointing to a directory that stores packages (*.svp)`
8	`files. buildidx will generate the index file and save it into the package`	8	`files. buildidx will generate the index file and save it into the package`
9	`repository.`	9	`repository.`
10		10
11	`requires php-zip`	11	`requires php-zip`
12		12
-		13	`14 feb 2022: packages are expected to have the *.svp extension`
13	`12 feb 2022: skip source packages from being processed (*.src.zip)`	14	`12 feb 2022: skip source packages from being processed (*.src.zip)`
14	`20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance`	15	`20 jan 2022: rewritten the code from ANSI C to PHP for easier maintenance`
15	`13 feb 2021: 'title' LSM field is no longer looked after`	16	`13 feb 2021: 'title' LSM field is no longer looked after`
16	`11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS`	17	`11 feb 2021: lsm headers are no longer checked, so it is compatible with the simpler lsm format used by SvarDOS`
17	`13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index`	18	`13 jan 2021: removed the identification line, changed CRC32 to bsum, not creating the listing.txt file and stopped compressing index`
18	`23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present`	19	`23 apr 2017: uncompressed index is no longer created, added CRC32 of zib (bin only) files, if present`
19	`28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)`	20	`28 aug 2016: listing.txt is always written inside the repo dir (instead of inside current dir)`
20	`27 aug 2016: accepting full paths to repos (starting with /...)`	21	`27 aug 2016: accepting full paths to repos (starting with /...)`
21	`07 dec 2013: rewritten buildidx in ANSI C89`	22	`07 dec 2013: rewritten buildidx in ANSI C89`
22	`19 aug 2013: add a compressed version of the index file to repos (index.gz)`	23	`19 aug 2013: add a compressed version of the index file to repos (index.gz)`
23	`22 jul 2013: creating a listing.txt file with list of packages`	24	`22 jul 2013: creating a listing.txt file with list of packages`
24	`18 jul 2013: writing the number of packaged into the first line of the lst file`	25	`18 jul 2013: writing the number of packaged into the first line of the lst file`
25	`11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files`	26	`11 jul 2013: added a switch to 7za to make it case insensitive when extracting lsm files`
26	`10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)`	27	`10 jul 2013: changed unzip calls to 7za (to handle cases when appinfo is compressed with lzma)`
27	`04 feb 2013: added CRC32 support`	28	`04 feb 2013: added CRC32 support`
28	`22 sep 2012: forked 1st version from FDUPDATE builder`	29	`22 sep 2012: forked 1st version from FDUPDATE builder`
29	`*/`	30	`*/`
30		31
31	`$PVER = "20220120";`	32	`$PVER = "20220214";`
32		33
33		34
34	`// computes the BSD sum of a file and returns it`	35	`// computes the BSD sum of a file and returns it`
35	`function file2bsum($fname) {`	36	`function file2bsum($fname) {`
36	`$result = 0;`	37	`$result = 0;`
37		38
38	`$fd = fopen($fname, 'rb');`	39	`$fd = fopen($fname, 'rb');`
39	`if ($fd === false) return(0);`	40	`if ($fd === false) return(0);`
40		41
41	`while (!feof($fd)) {`	42	`while (!feof($fd)) {`
42		43
43	`$buff = fread($fd, 1024 * 1024);`	44	`$buff = fread($fd, 1024 * 1024);`
44		45
45	`$slen = strlen($buff);`	46	`$slen = strlen($buff);`
46	`for ($i = 0; $i < $slen; $i++) {`	47	`for ($i = 0; $i < $slen; $i++) {`
47	`// rotr`	48	`// rotr`
48	`$result = ($result >> 1) \| ($result << 15);`	49	`$result = ($result >> 1) \| ($result << 15);`
49	`// add and truncate to 16 bits`	50	`// add and truncate to 16 bits`
50	`$result += ord($buff[$i]);`	51	`$result += ord($buff[$i]);`
51	`$result &= 0xffff;`	52	`$result &= 0xffff;`
52	`}`	53	`}`
53	`}`	54	`}`
54		55
55	`fclose($fd);`	56	`fclose($fd);`
56	`return($result);`	57	`return($result);`
57	`}`	58	`}`
58		59
59		60
60	`// reads file fil from zip archive z and returns its content, or false on error`	61	`// reads file fil from zip archive z and returns its content, or false on error`
61	`function read_file_from_zip($z, $fil) {`	62	`function read_file_from_zip($z, $fil) {`
62	`$zip = new ZipArchive;`	63	`$zip = new ZipArchive;`
63	`if ($zip->open($z, ZipArchive::RDONLY) !== true) {`	64	`if ($zip->open($z, ZipArchive::RDONLY) !== true) {`
64	`echo "ERROR: failed to open zip file '{$z}'\n";`	65	`echo "ERROR: failed to open zip file '{$z}'\n";`
65	`return(false);`	66	`return(false);`
66	`}`	67	`}`
67		68
68	`// load the appinfo/pkgname.lsm file`	69	`// load the appinfo/pkgname.lsm file`
69	`$res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);`	70	`$res = $zip->getFromName($fil, 8192, ZipArchive::FL_NOCASE);`
70		71
71	`$zip->close();`	72	`$zip->close();`
72	`return($res);`	73	`return($res);`
73	`}`	74	`}`
74		75
75		76
76	`// reads a LSM string and returns it in the form of an array`	77	`// reads a LSM string and returns it in the form of an array`
77	`function parse_lsm($s) {`	78	`function parse_lsm($s) {`
78	`$res = array();`	79	`$res = array();`
79	`for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {`	80	`for ($l = strtok($s, "\n"); $l !== false; $l = strtok("\n")) {`
80	`// the line is "token: value", let's find the colon`	81	`// the line is "token: value", let's find the colon`
81	`$colpos = strpos($l, ':');`	82	`$colpos = strpos($l, ':');`
82	`if (($colpos === false) \|\| ($colpos === 0)) continue;`	83	`if (($colpos === false) \|\| ($colpos === 0)) continue;`
83	`$tok = strtolower(trim(substr($l, 0, $colpos)));`	84	`$tok = strtolower(trim(substr($l, 0, $colpos)));`
84	`$val = trim(substr($l, $colpos + 1));`	85	`$val = trim(substr($l, $colpos + 1));`
85	`$res[$tok] = $val;`	86	`$res[$tok] = $val;`
86	`}`	87	`}`
87	`return($res);`	88	`return($res);`
88	`}`	89	`}`
89		90
90		91
91	`// *************** MAIN ROUTINE *******************************************`	92	`// *************** MAIN ROUTINE *******************************************`
92		93
93	`echo "SvarDOS repository index generator ver {$PVER}\n";`	94	`echo "SvarDOS repository index generator ver {$PVER}\n";`
94		95
95	`if (($_SERVER['argc'] != 2) \|\| ($_SERVER['argv'][1][0] == '-')) {`	96	`if (($_SERVER['argc'] != 2) \|\| ($_SERVER['argv'][1][0] == '-')) {`
96	`echo "usage: php buildidx.php repodir\n";`	97	`echo "usage: php buildidx.php repodir\n";`
97	`exit(1);`	98	`exit(1);`
98	`}`	99	`}`
99		100
100	`$repodir = $_SERVER['argv'][1];`	101	`$repodir = $_SERVER['argv'][1];`
101		102
102	`echo "building index for directory {$repodir}...\n";`	103	`echo "building index for directory {$repodir}...\n";`
103		104
104	`$pkgfiles = scandir($repodir);`	105	`$pkgfiles = scandir($repodir);`
105	`$pkglist = '';`	106	`$pkglist = '';`
106	`$pkgcount = 0;`	107	`$pkgcount = 0;`
107		108
108	`// iterate over each zip file`	109	`// iterate over each svp package`
109	`foreach ($pkgfiles as $zipfile) {`	110	`foreach ($pkgfiles as $zipfile) {`
110	`if (!preg_match('/.zip$/i', $zipfile)) continue; // skip non-zip files`	111	`if (!preg_match('/.svp$/i', $zipfile)) continue; // skip non-svp files`
111	`if (strchr($zipfile, '-')) {`	112	`if (strchr($zipfile, '-')) {`
112	`echo "skipping: {$zipfile}\n";`	113	`echo "skipping: {$zipfile}\n";`
113	`continue; // skip alt vers (like dosmid-0.9.2.zip)`	114	`continue; // skip alt vers (like dosmid-0.9.2.svp)`
114	`}`	115	`}`
115	`if (strstr($zipfile, '.src.zip')) {`	-
116	`echo "skipping source: {$zipfile}\n";`	-
117	`continue;`	-
118	`}`	-
119		-
120		116
121	`$path_parts = pathinfo($zipfile);`	117	`$path_parts = pathinfo($zipfile);`
122	`$pkg = strtolower($path_parts['filename']);`	118	`$pkg = strtolower($path_parts['filename']);`
123		119
124	`$zipfile_fullpath = realpath($repodir . '/' . $zipfile);`	120	`$zipfile_fullpath = realpath($repodir . '/' . $zipfile);`
125		121
126	`$lsm = read_file_from_zip($zipfile_fullpath, "appinfo/{$pkg}.lsm");`	122	`$lsm = read_file_from_zip($zipfile_fullpath, "appinfo/{$pkg}.lsm");`
127	`if ($lsm == false) {`	123	`if ($lsm == false) {`
128	`echo "ERROR: pkg {$z} does not contain an LSM file at the expected location\n";`	124	`echo "ERROR: pkg {$z} does not contain an LSM file at the expected location\n";`
129	`exit(1);`	125	`exit(1);`
130	`}`	126	`}`
131		127
132	`$lsmarray = parse_lsm($lsm);`	128	`$lsmarray = parse_lsm($lsm);`
133	`if (empty($lsmarray['version'])) {`	129	`if (empty($lsmarray['version'])) {`
134	`echo "ERROR: lsm file in {$zipfile} does not contain a version\n";`	130	`echo "ERROR: lsm file in {$zipfile} does not contain a version\n";`
135	`var_dump($lsmarray);`	131	`var_dump($lsmarray);`
136	`exit(1);`	132	`exit(1);`
137	`}`	133	`}`
138	`if (empty($lsmarray['description'])) {`	134	`if (empty($lsmarray['description'])) {`
139	`echo "ERROR: lsm file in {$zipfile} does not contain a description\n";`	135	`echo "ERROR: lsm file in {$zipfile} does not contain a description\n";`
140	`exit(1);`	136	`exit(1);`
141	`}`	137	`}`
142		138
143	`$pkglist .= "{$pkg}\t{$lsmarray['version']}\t{$lsmarray['description']}\t" . file2bsum($zipfile_fullpath) . "\n";`	139	`$pkglist .= "{$pkg}\t{$lsmarray['version']}\t{$lsmarray['description']}\t" . file2bsum($zipfile_fullpath) . "\n";`
144	`$pkgcount++;`	140	`$pkgcount++;`
145		141
146	`}`	142	`}`
147		143
148	`echo "DONE - processed " . $pkgcount . " zip files\n";`	144	`echo "DONE - processed " . $pkgcount . " svp packages\n";`
149		145
150	`file_put_contents($repodir . '/index.tsv', $pkglist);`	146	`file_put_contents($repodir . '/index.tsv', $pkglist);`
151		147
152	`exit(0);`	148	`exit(0);`
153		149
154	`?>`	150	`?>`
155		151