利用stream实现一个简单的http下载器

其实这个http下载器的功能已经相当完善了，支持：限速、post投递和上传、自定义http header、设置user agent、设置range和超时

而且它还不单纯只能下载http，由于使用了stream，所以也支持其他协议，你也可以用它来进行文件之间的copy、纯tcp下载等等。。

完整demo请参考：https://github.com/waruqi/tbox/wiki

stream.c

/* //////////////////////////////////////////////////////////////////////////////////////
 * includes
 */
#include "../demo.h"
 
/* //////////////////////////////////////////////////////////////////////////////////////
 * types
 */
typedef struct __tb_demo_context_t
{
  // verbose 
  tb_bool_t      verbose;
 
}tb_demo_context_t;
 
/* //////////////////////////////////////////////////////////////////////////////////////
 * func
 */
#ifdef TB_CONFIG_MODULE_HAVE_OBJECT
static tb_bool_t tb_demo_http_post_func(tb_size_t state, tb_hize_t offset, tb_hong_t size, tb_hize_t save, tb_size_t rate, tb_cpointer_t priv)
{
  // percent
  tb_size_t percent = 0;
  if (size > 0) percent = (tb_size_t)((offset * 100) / size);
  else if (state == TB_STATE_CLOSED) percent = 100;
 
  // trace
  tb_trace_i("post: %llu, rate: %lu bytes/s, percent: %lu%%, state: %s", save, rate, percent, tb_state_cstr(state));
 
  // ok
  return tb_true;
}
static tb_bool_t tb_demo_stream_head_func(tb_char_t const* line, tb_cpointer_t priv)
{
  tb_printf("response: %s\n", line);
  return tb_true;
}
static tb_bool_t tb_demo_stream_save_func(tb_size_t state, tb_hize_t offset, tb_hong_t size, tb_hize_t save, tb_size_t rate, tb_cpointer_t priv)
{
  // check
  tb_demo_context_t* context = (tb_demo_context_t*)priv;
  tb_assert_and_check_return_val(context, tb_false);
 
  // print verbose info
  if (context->verbose) 
  {
    // percent
    tb_size_t percent = 0;
    if (size > 0) percent = (tb_size_t)((offset * 100) / size);
    else if (state == TB_STATE_CLOSED) percent = 100;
 
    // trace
    tb_printf("save: %llu bytes, rate: %lu bytes/s, percent: %lu%%, state: %s\n", save, rate, percent, tb_state_cstr(state));
  }
 
  // ok
  return tb_true;
}
 
/* //////////////////////////////////////////////////////////////////////////////////////
 * globals
 */
static tb_option_item_t g_options[] = 
{
  {'-',  "gzip",     TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "enable gzip"        }
,  {'-',  "no-verbose",  TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "disable verbose info"   }
,  {'d',  "debug",    TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "enable debug info"     }
,  {'k',  "keep-alive",  TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "keep alive"        }
,  {'h',  "header",    TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_CSTR,    "the custem http header"  }
,  {'-',  "post-data",  TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_CSTR,    "set the post data"     }
,  {'-',  "post-file",  TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_CSTR,    "set the post file"     }
,  {'-',  "range",    TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_CSTR,    "set the range"       }
,  {'-',  "timeout",   TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_INTEGER,   "set the timeout"      }
,  {'-',  "limitrate",  TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_INTEGER,   "set the limitrate"     }
,  {'h',  "help",     TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "display this help and exit"}
,  {'-',  "url",     TB_OPTION_MODE_VAL,     TB_OPTION_TYPE_CSTR,    "the url"          }
,  {'-',  tb_null,    TB_OPTION_MODE_MORE,    TB_OPTION_TYPE_NONE,    tb_null           }
 
};
 
/* //////////////////////////////////////////////////////////////////////////////////////
 * main
 */
tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv)
{
  // done
  tb_option_ref_t   option = tb_null;
  tb_stream_ref_t   istream = tb_null;
  tb_stream_ref_t   ostream = tb_null;
  tb_stream_ref_t   pstream = tb_null;
  do
  {
    // init option
    option = tb_option_init("stream", "the stream demo", g_options);
    tb_assert_and_check_break(option);
   
    // done option
    if (tb_option_done(option, argc - 1, &argv[1]))
    {
      // debug & verbose
      tb_bool_t debug = tb_option_find(option, "debug");
      tb_bool_t verbose = tb_option_find(option, "no-verbose")? tb_false : tb_true;
     
      // done url
      if (tb_option_find(option, "url")) 
      {
        // init istream
        istream = tb_stream_init_from_url(tb_option_item_cstr(option, "url"));
        tb_assert_and_check_break(istream);
   
        // ctrl http
        if (tb_stream_type(istream) == TB_STREAM_TYPE_HTTP) 
        {
          // enable gzip?
          if (tb_option_find(option, "gzip"))
          {
            // auto unzip
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break;
 
            // need gzip
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break;
          }
 
          // enable debug?
          if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, debug? tb_demo_stream_head_func : tb_null)) break;
 
          // custem header?
          if (tb_option_find(option, "header"))
          {
            // init
            tb_string_t key;
            tb_string_t val;
            tb_string_init(&key);
            tb_string_init(&val);
 
            // done
            tb_bool_t      k = tb_true;
            tb_char_t const*  p = tb_option_item_cstr(option, "header");
            while (*p)
            {
              // is key?
              if (k)
              {
                if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++);
                else if (*p == ':') 
                {
                  // skip ':'
                  p++;
 
                  // skip space
                  while (*p && tb_isspace(*p)) p++;
 
                  // is val now
                  k = tb_false;
                }
                else p++;
              }
              // is val?
              else
              {
                if (*p != ';') tb_string_chrcat(&val, *p++);
                else
                {
                  // skip ';'
                  p++;
 
                  // skip space
                  while (*p && tb_isspace(*p)) p++;
 
                  // set header
                  if (tb_string_size(&key) && tb_string_size(&val))
                  {
                    if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
                    if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
                  }
 
                  // is key now
                  k = tb_true;
 
                  // clear key & val
                  tb_string_clear(&key);
                  tb_string_clear(&val);
                }
              }
            }
 
            // set header
            if (tb_string_size(&key) && tb_string_size(&val))
            {
              if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
              if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
            }
 
            // exit 
            tb_string_exit(&key);
            tb_string_exit(&val);
          }
 
          // keep alive?
          if (tb_option_find(option, "keep-alive"))
          {
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break;
          }
 
          // post-data?
          if (tb_option_find(option, "post-data"))
          {
            tb_char_t const*  post_data = tb_option_item_cstr(option, "post-data");
            tb_hize_t      post_size = tb_strlen(post_data);
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break;
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
            if (debug) tb_printf("post: %llu\n", post_size);
          }
          // post-file?
          else if (tb_option_find(option, "post-file"))
          {
            tb_char_t const* url = tb_option_item_cstr(option, "post-file");
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break;
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
            if (debug) tb_printf("post: %s\n", url);
          }
        }
 
        // set range
        if (tb_option_find(option, "range"))
        {
          tb_char_t const* p = tb_option_item_cstr(option, "range");
          if (p)
          {
            // the bof
            tb_hize_t eof = 0;
            tb_hize_t bof = tb_atoll(p);
            while (*p && tb_isdigit(*p)) p++;
            if (*p == '-')
            {
              p++;
              eof = tb_atoll(p);
            }
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break;
          }
        }
 
        // set timeout
        if (tb_option_find(option, "timeout"))
        {
          tb_size_t timeout = tb_option_item_uint32(option, "timeout");
          if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_SET_TIMEOUT, timeout)) break;
        }
 
        // print verbose info
        if (verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(option, "url"));
 
        // open istream
        if (!tb_stream_open(istream)) 
        {
          // print verbose info
          if (verbose) tb_printf("open: %s\n", tb_state_cstr(tb_stream_state(istream)));
          break;
        }
 
        // print verbose info
        if (verbose) tb_printf("open: ok\n");
 
        // init ostream
        if (tb_option_find(option, "more0"))
        {
          // the path
          tb_char_t const* path = tb_option_item_cstr(option, "more0");
 
          // init
          ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
 
          // print verbose info
          if (verbose) tb_printf("save: %s\n", path);
        }
        else
        {
          // the name
          tb_char_t const* name = tb_strrchr(tb_option_item_cstr(option, "url"), '/');
          if (!name) name = tb_strrchr(tb_option_item_cstr(option, "url"), '\\');
          if (!name) name = "/stream.file";
 
          // the path
          tb_char_t path[TB_PATH_MAXN] = {0};
          if (tb_directory_curt(path, TB_PATH_MAXN))
            tb_strcat(path, name);
          else break;
 
          // init file
          ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
 
          // print verbose info
          if (verbose) tb_printf("save: %s\n", path);
        }
        tb_assert_and_check_break(ostream);
 
        // the limit rate
        tb_size_t limitrate = 0;
        if (tb_option_find(option, "limitrate"))
          limitrate = tb_option_item_uint32(option, "limitrate");
 
        // save it
        tb_hong_t      save = 0;
        tb_demo_context_t  context = {0}; 
        context.verbose   = verbose;
        if ((save = tb_transfer_done(istream, ostream, limitrate, tb_demo_stream_save_func, &context)) < 0) break;
      }
      else tb_option_help(option);
    }
    else tb_option_help(option);
 
  } while (0);
 
  // exit pstream
  if (pstream) tb_stream_exit(pstream);
  pstream = tb_null;
 
  // exit istream
  if (istream) tb_stream_exit(istream);
  istream = tb_null;
 
  // exit ostream
  if (ostream) tb_stream_exit(ostream);
  ostream = tb_null;
 
  // exit option
  if (option) tb_option_exit(option);
  option = tb_null;
 
  return 0;
}
#else
tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv)
{
  return 0;
}
#endif

以上所述就是本文的全部内容了，希望大家能够喜欢。