BlitzGet MaxDeluxe


Tweet blitzmax networking code-archives
(Posted 4 months ago) RonTek

This is just a BlitzMax implementation of an old file downloader for Blitz2D/3D. I've cleaned things up and included support for redirected URLs. This hasn't been heavily tested, but seems to work fine for standard HTTP downloads, files that aren't found and files that are redirected to new URLs.

Author: BlitzSupport

' SuperStrict

Function BlitzGet:Int (url:String, savepath:String, savefile:String)

    If Not url Then Return False

    Local success:Int = False   ' File downloaded OK...
    Local done:Int = False      ' Exit download loop (for retries, etc)...

    Local host:String
    Local file:String

    Local bytestoread:Int
    Local date:String
    Local server:String
    Local contenttype:String
    Local location:String
    Local pos:Int

    Print ""
    Print "Downloading..."
    Print ""

    Repeat

        If Left (url, 7) = "http://"
            url= Right (url, Len (url) - 7)
        EndIf

        Local slash:Int = Instr (url, "/")

        If slash
            host = Left (url, slash - 1)
            file = Right (url, Len (url) - slash + 1)
        Else
            host = url
            file = "/"
        EndIf

        If Right (savepath, 1) <> "\" And Right (savepath, 1) <> "/" Then savepath = savepath + "\"

        If savefile = ""

            If file = "/"
                savefile = "Unknown file.txt"
            Else

                Local findslash:Int
                Local testforslash:String

                For findslash = Len (file) To 1 Step - 1
                    testforslash = Mid (file, findslash, 1)
                    If testforslash = "/"
                        savefile = Right (file, Len (file) - findslash)
                        Exit
                    EndIf
                Next

                If savefile = "" Then savefile = "Unknown file.txt"

            EndIf

        EndIf

        Local http:TSocket = CreateTCPSocket ()

        If http

            If ConnectSocket (http, HostIp (host), 80)

                Local www:TStream = CreateSocketStream (http)

                WriteLine www, "GET " + file + " HTTP/1.1" ' "GET /" gets default page...
                WriteLine www, "Host: " + host
                WriteLine www, "User-Agent: BlitzGet Deluxe"
                WriteLine www, "Accept: */*"
                WriteLine www, ""

                Local response:String = ReadLine (www)

                Print "Server response: " + response

                Local replycode:String

                If Left (response, 5) = "HTTP/"
                    pos = Instr (response, " ")
                    replycode = Mid (response, pos + 1, 3)
                EndIf

                Local header:String

                Repeat

                    header = ReadLine (www)

                    Local reply:String = ""

                    pos = Instr (header, ": ")

                    If pos
                        reply = Left (header, pos + 1)
                    EndIf

                    Select Lower (reply)
                        Case "content-length: "
                            bytestoread = Int (Right (header, Len (header) - Len (reply)))
                        Case "location: "
                            location = Right (header, Len (header) - Len (reply))
                    End Select

                    If header Then Print header ' Skip blank line (if header = "" then nothing is printed)...

                Until header$ = "" Or (Eof (www))

                Select replycode$

                    Case "200" ' File found...

                        Print "Downloading file..."

                        Local save:TStream = WriteFile (savepath + savefile)

                        If save

                            Local readwebfile:Int

                            ' Crude download routine!

                            CopyBytes www, save, bytestoread

                            CloseFile save

                            ' Fully downloaded, ie. same size?

                            If FileSize (savepath + savefile) = bytestoread
                                success = True
                            EndIf

                            done = True

                        Else
                            Print "Failed to create local file!"
                        EndIf

                    Case "404" ' File Not found...

                        Print "File not found"
                        done = True

                    Case "301" ' File permanently moved...

                        url = location

                    Case "302" ' File temporarily moved...

                        url = location

                    Case "303" ' File moved...

                        url = location

                    Case "307" ' Naughty...

                        url = location

                End Select

            EndIf

            CloseSocket http

        EndIf

        ' If 'done' is still false, go back to the start with new URL (from '30*' responses)...

    Until done

    Return success

End Function

' This URL has worked for years...

Local download:String = "http://www.google.com/images/title_homepage4.gif"          ' Google homepage logo

' Test error 404, file not found...
' download:String    = "http://www.hi-toro.com/mp3/diffusion.mp3"                   ' Old music (now gone)...

' Test error 303, file moved example (works as of 28 Aug 2009)...
' download:String = "http://www.rentnet.com/" ' Redirects to http://www.move.com/apartments/main.aspx

Local download_path:String = CurrentDir ()

If BlitzGet (download, download_path, "")
    Print ""
    Print "File downloaded successfully! Check " + download_path
Else
    Print ""
    Print "Download failed!"
EndIf
(Posted 3 months ago) druggedbunny983 commented:

I wrote this and was very proud of the server response handling! However, it's almost 100% useless today, since most sites use SSL (HTTPS).

SSL is incredibly complex and way out of the scope of a hobbiest coder to implement, so interacting with a server like the above isn't really an option for most of us.

I now use variations of the horrible code below for BMX-NG, depending on what I need to do. As you can see, the simple act of downloading a file is now subject to a mass of dependencies and third-party files (cacert.pem) that need to be distributed with your application... sheer misery. (This is a comment on modern systems, not Brucey's work!)

IMPORTANT: NOTE THE COMMENTS AT THE START! It will silently fail otherwise.


' IMPORTANT!

' You need to download and place a copy
' of "cacert.pem" in same folder...

' From https://curl.haxx.se/docs/caextract.html
' https://curl.haxx.se/ca/cacert.pem

' Depends on:

Rem

    https://github.com/maxmods/bah.mod

    mod\bah.mod\libcurl.mod
    mod\bah.mod\libssh2.mod
    mod\bah.mod\mbedtls.mod

EndRem

Framework BaH.libcurl

Import BRL.StandardIO
Import BRL.FileSystem

Function SaveHTML (base_url:String, save_as:String)

    Local curl:TCurlEasy = TCurlEasy.Create()

    'curl.setOptInt(CURLOPT_VERBOSE, 1)
    curl.setOptInt(CURLOPT_FOLLOWLOCATION, 1)
    curl.setWriteString()

    curl.setOptString(CURLOPT_CAINFO, "cacert.pem")

    curl.setOptString(CURLOPT_URL, base_url)

    Local res:Int = curl.perform()

    curl.cleanup()

    SaveString (curl.ToString (), save_as)

EndFunction

SaveHTML ("https://duckduckgo.com/", MilliSecs () + "_test.html")
(Posted 3 months ago) RonTek commented:

Hey DruggedBunny, Welcome to the Forums! smiley:

Thanks for sharing an updated SSL version! I actually worked on a BlitzMax SSL project once using LibCurl and it is for a game that I am trying to release.

Enjoy your stay and hope you'll share more Blitz stuff!

Reply To Topic (minimum 10 characters)

Please log in to reply