#! /bin/sh
# htget --- Get raw HTML from URL

# Copyright (C) 1995, 1996 Noah S. Friedman

# Author: Noah Friedman <friedman@prep.ai.mit.edu>
# Created: 1995-11-07

# $Id: htget,v 1.7 1996/07/15 01:47:58 friedman Exp $

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, you can either send email to this
# program's maintainer or write to: The Free Software Foundation,
# Inc.; 59 Temple Place, Suite 330; Boston, MA 02111-1307, USA.

# Commentary:

# This script requires the `tcpconnect' program.

# Code:

progname=`echo "$0" | sed -e 's/[^\/]*\///g'`

bq='`'
eq="'"

usage="Usage: $progname {options} [URL]

Options are:
-D, --debug                  Turn on shell debugging ($bq${bq}set -x$eq$eq).
-h, --help                   You're looking at it.
-o, --only-headers           Output reply headers only (no html).
-p, --protocol       PROT    Use HTTP protocol PROT.  Default is 1.0.
-s, --show-headers           Prepend reply headers to html output.
-v, --verbose                Be verbose.
"

# Initialize variables.
# Don't use `unset' since old bourne shells don't have this command.
# Instead, assign them an empty value.
debug=
headers=
verbose=
protocol=1.0

# Usage: eval "$getopt"; value=$optarg
# or     optarg_optional=t; eval "$getopt"; value=$optarg
#
# This function automatically shifts the positional args as appropriate.
# The argument to an option is optional if the variable `optarg_optional'
# is non-empty.  Otherwise, the argument is required and getopt will cause
# the program to exit on an error.  optarg_optional is reset to be empty
# after every call to getopt.  The argument (if any) is stored in the
# variable `optarg'.
#
# Long option syntax is `--foo=bar' or `--foo bar'.  2nd argument
# won't get used if first long option syntax was used.
#
# Note: because of broken bourne shells, using --foo=bar syntax can
# actually screw the quoting of args that end with trailing newlines.
# Specifically, most shells strip trailing newlines from substituted
# output, regardless of quoting.
getopt='
  {
    optarg=
    case "$1" in
      --*=* )
        optarg=`echo "$1" | sed -e "1s/^[^=]*=//"`
        shift
       ;;
      * )
        case ${2+set} in
          set )
            optarg="$2"
            shift
            shift
           ;;
          * )
            case "$optarg_optional" in
              "" )
                case "$1" in
                  --*=* ) option=`echo "$1" | sed -e "1s/=.*//;q"` ;;
                  * ) option="$1" ;;
                esac
                exec 1>&2
                echo "$progname: option $bq$option$eq requires argument."
                echo "$progname: use $bq--help$eq to list option syntax."
                exit 1
               ;;
           esac
           shift
         ;;
        esac
     ;;
    esac
    optarg_optional=
  }'

# Parse command line arguments.
# Make sure that all wildcarded options are long enough to be unambiguous.
# It's a good idea to document the full long option name in each case.
# Long options which take arguments will need a `*' appended to the
# canonical name to match the value appended after the `=' character.
while : ; do
  case $# in 0) break ;; esac
  case "$1" in
    -D | --debug | --d* )
      debug=t
      shift
     ;;
    -h | --help | --h* )
      echo "$usage" 1>&2
      exit 0
     ;;
    -o | --only-headers | --o* )
      headers=only
      shift
     ;;
    -p | --protocol* | --p* )
      eval "$getopt"
      protocol="$optarg"
     ;;
    -s | --show-headers | --s* )
      headers=t
      shift
     ;;
    -v | --verbose | --v* )
      verbose=-v
      shift
     ;;
    -- )     # Stop option processing
      shift
      break
     ;;
    -? | --* )
      case "$1" in
        --*=* ) arg=`echo "$1" | sed -e 's/=.*//'` ;;
        * )     arg="$1" ;;
      esac
      exec 1>&2
      echo "$progname: unknown or ambiguous option $bq$arg$eq"
      echo "$progname: Use $bq--help$eq for a list of options."
      exit 1
     ;;
    -??* )
      # Split grouped single options into separate args and try again
      optarg="$1"
      shift
      set fnord `echo "x$optarg" | sed -e 's/^x-//;s/\(.\)/-\1 /g'` ${1+"$@"}
      shift
     ;;
    * )
      break
     ;;
  esac
done

case "$debug" in t ) set -x ;; esac

case "$1" in
  '' )
    echo "$usage" 1>&2
    exit 1
   ;;
  http://* ) : ;;
  http:* )
    echo "$progname: malformed URL $bq$1$eq." 1>&2
    exit 1
   ;;
  *: )
    echo "$progname: Can only retrieve URLs of type ${bq}http$eq." 1>&2
    exit 1
esac

# Args should be: fnord [host] [port] [file]
set fnord `echo "$1" | sed -ne 's/^http:\/\///
                                /^[^/]*:[0-9]/!s/\([^/]*\)/\1:80/
                                s/\([^:]*\):\([0-9]*\)\(\/.*\)/\1 \2 \3/p
                               '`
shift

case "$protocol" in
  0.9 )
    case "$headers" in
      '' ) : ;;
      *  ) echo "$progname: warning: HTTP/0.9 cannot obtain headers." 1>&2 ;;
    esac
   ;;
  *   ) req="$req HTTP/$protocol" ;;
esac

export verbose
export headers
export protocol

exec ${PERL-perl} - ${1+"$@"} <<'__EOF__'

&main (@ARGV);

sub main
{
  local ($verbose)  = $ENV{'verbose'};
  local ($headers)  = $ENV{'headers'};
  local ($protocol) = $ENV{'protocol'};

  local ($host, $port, $file) = @_;
  local ($req, $prot);
  local ($content_length, $readcount, $lastpercentage) = (0, 0, 0);
  local ($in_headers) = 1;

  &connect ($host, $port);

  $req = 'GET';
  if ($ENV{'headers'} eq 'only') { $req = 'HEAD'; }

  $prot = " $ENV{protocol}";
  if ($prot eq ' 0.9') { $prot = ''; }

  print PARENTWRITE "$req $file$prot\r\n\r\n";
  close (PARENTWRITE);

  while (<PARENTREAD>)
    {
      if ($in_headers)
        {
          s/\r\n$/\n/o;
          if ($verbose ne '' && $_ =~ /^content-length:[ \t]/io)
            {
              $content_length = $_;
              $content_length =~ s/^content-length:[ \t]+//io;
              chop $content_length;
            }

          print if ($headers eq "t" || $headers eq "only");
          $in_headers = 0 if ($_ eq "\n");
        }
      else
        {
          if ($content_length > 0)
            {
              local ($percentage);

              $readcount += length ($_);
              $percentage = int (($readcount / $content_length) * 100);

              if ($percentage > ($lastpercentage + 5))
                {
                  printf (STDERR "%-2.2d%% ", $percentage);
                  $lastpercentage = $percentage;
                }
            }
          print;
        }
    }
  close (PARENTREAD);

  if ($lastpercentage > 0)
    {
      if ($lastpercentage < 100)
        {
          print STDERR "100%";
        }
      print STDERR "\n";
    }
}

sub connect
{
  local ($host, $port) = @_;
  local (@args) = ('tcpconnect', $host, $port);

  if ($ENV{'verbose'} ne '')
    {
      splice (@args, 1, 0, $ENV{'verbose'});
    }
  &spawn (@args);
}

sub spawn
{
  local ($pid);

  pipe (CHILDREAD, PARENTWRITE);
  pipe (PARENTREAD, CHILDWRITE);

  select (CHILDREAD);   $| = 1;
  select (CHILDWRITE);  $| = 1;
  select (PARENTREAD);  $| = 1;
  select (PARENTWRITE); $| = 1;
  select (STDOUT);

  $pid = fork;
  die if (! defined ($pid));

  if ($pid == 0)
    {
      close (PARENTREAD);
      close (PARENTWRITE);

      open (STDIN, "<&CHILDREAD");
      open (STDOUT, ">&CHILDWRITE");
      close (CHILDREAD);
      close (CHILDWRITE);
      exec (@_);
    }
  else
    {
      close (CHILDREAD);
      close (CHILDWRITE);
    }
}

__EOF__


# htget ends here