nodejs_dist_list/parse_dist.php

   1 <?php
   2
   3 $baseurl  = "http://nodejs.org/dist";
   4
   5 // we scrape nodejs.org/dist site looking for versions that exist so we can construct a meta data database
   6 // so our lxc will be able to find versions in a more appropriate way
   7
   8 // first, get the main site
   9
  10 $fver = array();
  11 $fvern = 0;
  12 global $fver, $baseurl, $fvern;
  13
  14
  15 echo "Getting base list from $baseurl\n";
  16 $distbase = file_get_contents($baseurl);
  17
  18 // get all links that are directories
  19 preg_match_all('/<a [^>]*href="(.+)"/', $distbase, $firstlist);
  20
  21 $versions = array();
  22
  23 foreach($firstlist[1] as $chk) {
  24         //echo "Got: $chk\n";
  25         if(preg_match('/v[0-9]+.[0-9]+.[0-9]+\//', $chk)) {
  26                 if(preg_match('/v([0-9]+.[0-9]+.[0-9]+)\//', $chk, $vlist)) {
  27                         $versions[$vlist[1]] = $chk;
  28                 }
  29         }
  30 }
  31
  32 foreach($versions as $key => $val) {
  33         // now we get each version and parse whats in its directory
  34         echo "Checking $key => $baseurl/$val\n";
  35
  36         //if(preg_match("/0\.8\.[1].*/", $key)) {
  37                 $urlnow = "$baseurl/$val";
  38                 $vers_dir = file_get_contents($urlnow);
  39
  40                 preg_match_all('/<a [^>]*href="(.+)"/', $vers_dir, $fileslist);
  41
  42                 checkFilesList(rtrim($val, "/"), $key, $fileslist);
  43         //}
  44 }
  45
  46
  47 // done!
  48 //echo "and fver is now\n";
  49 //var_dump($fver);
  50
  51 echo "dumping to versions file\n";
  52 $versdate = date("Ymd");
  53 $versfilename = "./versions_file.$versdate";
  54 $versfile = fopen("$versfilename", "w");
  55 if($versfile !== false) {
  56         fwrite($versfile, "baseurl:$baseurl\n");
  57         foreach($fver as $line) {
  58                 fwrite($versfile, "$line\n");
  59         }
  60         fclose($versfile);
  61
  62         // compress teh file
  63         system("gzip -c $versfilename > $versfilename.gz");
  64
  65         // create a current versions file
  66         $cv = fopen("current_version", "w");
  67         if($cv !== false) {
  68                 fwrite($cv, "version:$versdate\n");
  69                 fclose($cv);
  70         }
  71 } else {
  72         echo "Failed to open vers file!\n";
  73 }
  74
  75
  76 function checkFilesList($url, $vers, $fileslist) {
  77         global $fver, $baseurl, $fvern;
  78
  79         echo "Checking $url, $vers\n";
  80
  81         $shalist = array();
  82
  83         foreach($fileslist[1] as $fname) {
  84                 $is_echo = false;
  85                 $flurl = "";
  86                 $arch = "";
  87                 $plat = "";
  88                 $thissha = "";
  89
  90                 if($fname == "SHASUMS.txt") {
  91                         //$shas = file_get_
  92                         $shas_r = file_get_contents("$baseurl/$url/SHASUMS.txt");
  93                         $shalist = explode("\n", $shas_r);
  94                         foreach($shalist as $ssha) {
  95                                 $fsha = preg_split("/[ \t]+/", $ssha);
  96                                 if(isset($fsha[1])) $shalist[$fsha[1]] = $fsha[0];
  97                         }
  98                         //echo "Shasums file: $fname\n";
  99                         //var_dump($shalist);
 100                 } else if($fname == "MD5SUMS.txt") {
 101                         // nothing yet
 102                 } else  if(preg_match("/.*linux.*64\.tar\.gz/", $fname)) {
 103                         //echo "64 bit version for linux: $fname\n";
 104                         $thissha = "-";
 105                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 106                         $is_echo = true;
 107                         $plat = "linux";
 108                         $arch = "x86_64";
 109                 } else if(preg_match("/.*linux.*86\.tar\.gz/", $fname)) {
 110                         //echo "32 bit version for linux: $fname\n";
 111                         $thissha = "-";
 112                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 113                         //echo "$vers:$url/$fname:linux:i686:$thissha\n";
 114                         $is_echo = true;
 115                         $plat = "linux";
 116                         $arch = "i686";
 117                 } else  if(preg_match("/.*darwin.*86\.tar\.gz/", $fname)) {
 118                         //echo "32 bit version for darwin: $fname\n";
 119                         $thissha = "-";
 120                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 121                         //echo "$vers:$url/$fname:darwin:i686:$thissha\n";
 122                         $is_echo = true;
 123                         $plat = "darwin";
 124                         $arch = "i686";
 125                 } else if(preg_match("/.*darwin.*64\.tar\.gz/", $fname)) {
 126                         //echo "64 bit version for darwin: $fname\n";
 127                         $thissha = "-";
 128                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 129                         //echo "$vers:$url/$fname:darwin:x86_64:$thissha\n";
 130                         $is_echo = true;
 131                         $plat = "darwin";
 132                         $arch = "x86_64";
 133                 } else if(preg_match("/.*sunos.*64\.tar\.gz/", $fname)) {
 134                         // echo "64 bit version for sunos: $fname\n";
 135                         $thissha = "-";
 136                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 137                         //echo "$vers:$url/$fname:sunos:x86_64:$thissha\n";
 138                         $is_echo = true;
 139                         $plat = "sunos";
 140                         $arch = "x86_64";
 141                 } else if(preg_match("/.*sunos.*86\.tar\.gz/", $fname)) {
 142                         //echo "32 bit version for sunos: $fname\n";
 143                         $thissha = "-";
 144                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 145                         //echo "$vers:$url/$fname:sunos:i686:$thissha\n";
 146                         $is_echo = true;
 147                         $plat = "sunos";
 148                         $arch = "i686";
 149                 } else if(preg_match("/.*linux-arm.*\.tar\.gz/", $fname)) {
 150                         // echo "arm for linux: $fname\n";
 151                         $thissha = "-";
 152                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 153                         //echo "$vers:$url/$fname:linux:armhf:$thissha\n";
 154                         $is_echo = true;
 155                         $plat = "linux";
 156                         $arch = "armhf";
 157                 } else if(preg_match("/node-v[0-9]+\.[0-9]+\.[0-9]+\.tar\.gz/", $fname)) {
 158                         $thissha = "-";
 159                         if(isset($shalist[$fname])) $thissha = $shalist[$fname];
 160                         //echo "$vers:$url/$fname:linux:armhf:$thissha\n";
 161                         $is_echo = true;
 162                         $plat = "src";
 163                         $arch = "-";
 164                 }
 165
 166                 if($is_echo) {
 167                         $fver[$fvern++] = "$vers:$plat:$arch:$url/$fname:$thissha";
 168                 }
 169         }
 170 }
 171 ?>