Get Geodata (latitude, longitude, etc.) in Python

I let the API keys empty, 'cause I don't want to expose my keys. You need to register to infodb.com and google to get API keys !!


#!/usr/bin/python

import os
import sys
import socket
import httplib
import xml.dom.minidom as minidom
from googlemaps import GoogleMaps


""" Get googlemap API module from: http://pypi.python.org/pypi/googlemaps/1.0.2

    To get Googlemap API Key:
    https://code.google.com/apis/console/?pli=1#project:255524978890:access

"""

lat=38.4193
lon=-122.6978
myip = "99.9.21x.xx"  (hidden for the purpose of this blog)
debug=0


#time

lat2timeURL="www.earthtools.org"
#lat2timeFile="/timezone/" + str(lat) + '/' + str(lon)

lat2heightURL="www.earthtools.org"
#lat2heightFile="/height/" + str(lat) + "/" + str(lon)

infodb_apikey = "get your own key"
gmap_apikey = "get your own key"

ip2lat2URL = "api.ipinfodb.com"
ip2lat2File = "/v3/ip-city/?key=" + infodb_apikey + "&format=xml&ip="



def getText(nodeList):
    rc = []
    for node in nodeList:
        if node.nodeType == node.TEXT_NODE:
            rc.append(node.data)
    return ''.join(rc)

def getXmlElementFromObj(obj,  name):
    objList = obj.getElementsByTagName(name)[0]
    if objList:
        return getText(objList.childNodes)
    else:
        return ""


def DisplayUsage():
    sys.stderr.write("\nUsage: %s <address in double-quotes>\n\n" % sys.argv[0])


def EarthData(address):
    try:
        gmaps = GoogleMaps(gmap_apikey)
        lat, lon = gmaps.address_to_latlng(address)
    except:
        sys.stderr.write("\nUnable to query GoogleMaps\n")
        sys.exit(1)

    lat2timeFile="/timezone/" + str(lat) + '/' + str(lon)
    lat2heightFile="/height/" + str(lat) + "/" + str(lon)

    conn = httplib.HTTPConnection(lat2timeURL)
    conn.request("GET", lat2timeFile)
    resp = conn.getresponse()
    if resp.status == 200:
        data = resp.read()
        if debug:
            print data
        xml = minidom.parseString(data)
        timezoneObj = xml.getElementsByTagName("timezone")
        for tmObj in timezoneObj:
            nodes = tmObj.childNodes
            version = getXmlElementFromObj(tmObj, "version")
            lat = getXmlElementFromObj(tmObj, "latitude")
            lon = getXmlElementFromObj(tmObj, "longitude")
            localtime = getXmlElementFromObj(tmObj, "localtime")
            isotime = getXmlElementFromObj(tmObj, "isotime")
            utctime = getXmlElementFromObj(tmObj, "utctime")
            #print "version=%s" % version
            if debug:
                print "latitude  : %s" % lat
                print "longitude : %s" % lon
                print "localtime : %s" % localtime
    conn.close()

    conn = httplib.HTTPConnection(lat2heightURL)
    conn.request("GET", lat2heightFile)
    resp = conn.getresponse()
    if resp.status == 200:
        data = resp.read()
        if debug:
            print data
        xml = minidom.parseString(data)
        hObj = xml.getElementsByTagName("height")
        for h in hObj:
            meter = getText(h.getElementsByTagName("meters")[0].childNodes)
            feet = getText(h.getElementsByTagName("feet")[0].childNodes)
            if debug:
                print "Sea-level : %s meters = %s feet" % (meter, feet)

    conn.close()
    return (lat, lon, localtime, meter, feet)


def GetPublicIp(name):
    myip = str(socket.gethostbyname(name))

    iplatURL="api.hostip.info"
    ip2latFile="/?ip=" + myip + "&position=true"
    if debug:
        print "IP Address: %s" % myip
    ip2lat2File += myip
    conn = httplib.HTTPConnection(ip2lat2URL)
    conn.request("GET", ip2lat2File)
    resp = conn.getresponse()
    if resp.status == 200:
        data = resp.read()
        xml = minidom.parseString(data)
        #print data
        locObj = xml.getElementsByTagName("Response")
        for loc in locObj:
            nodes = loc.childNodes
            status = getXmlElementFromObj(loc,"statusCode")
            if status == "OK":
                lat = getXmlElementFromObj(loc, "latitude")
                lon = getXmlElementFromObj(loc, "longitude")
                countryCode = getXmlElementFromObj(loc, "countryCode")
                countryName = getXmlElementFromObj(loc, "countryName")
                regionName = getXmlElementFromObj(loc, "regionName")
                cityName = getXmlElementFromObj(loc, "cityName")
                zipCode = getXmlElementFromObj(loc, "zipCode")
                timezone = getXmlElementFromObj(loc, "timeZone")
                print "Address   : %s %s, %s %s" % (cityName, str(zipCode), regionName, countryName )
    conn.close()

"============================== MAIN =================================="
if __name__ == "__main__":
    if len(sys.argv) < 2:
        DisplayUsage()
        sys.exit(1)
    else:
        addr = sys.argv[1]
        print "Querying %s" % addr
        (lat, lon, ltime, meter, feet) = EarthData(addr)

    print "========================="
    print "Address   : %s" % addr
    print "Latitude  : %s" % lat
    print "Longitude : %s" % lon
    print "Local-time: %s" % ltime
    print "Sea-level : %s meters (%s feet)" % (meter,  feet)

#    print "ip2Lat2=%s%s" % (ip2lat2URL, ip2lat2File)


Example (assume the script above is saved as "earthdata.py"):


bash-4.2$ ./earthdata.py "1450 N McDowell Blvd, Petaluma, CA 94954"
Querying 1450 N McDowell Blvd, Petaluma, CA 94954
=========================
Address   : 1450 N McDowell Blvd, Petaluma, CA 94954
Latitude  : 38.279534
Longitude : -122.6709223
Local-time: 21 Oct 2012 16:47:33
Sea-level : 12 meters (39.4 feet)
bash-4.2$ 

Message-based Client-Server

Server


#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include <pthread.h>
#include <errno.h>
#include <sys/msg.h>
#include "msgdefs.h"


#define OK      0
#define LOG_ERROR(msg) { printf("%s:%u: %s %s\n", \
                __FUNCTION__, __LINE__, msg, \
                strerror(errno)); }
int msgQueue; 


int CreateMsgQueue(long id)
{
    int msgId;
    key_t key;
    struct msqid_ds ds;

    key = ftok("msgserv.c", id);
    msgId = msgget(key, 0644 | IPC_CREAT);
    if (msgId < 0)
    {
        LOG_ERROR("msgget");
    }
    if (EEXIST == errno)
    {
        // reuse
        msgctl(msgId, IPC_STAT, &ds);
        key = ftok("msgserv.key", id);
        msgId = msgget(key, 0777 | IPC_SET);
    }
    return msgId;
}


int SendMsg(void *msg, size_t msgsz, int msgflg)
{
    Msg_t *pMsg;
    char *pData;
    int retval = 0;
    
    pMsg = malloc(sizeof(Msg_t) + msgsz);
    pMsg->hdr.msgType = 1;
    pMsg->hdr.dataSize = msgsz;
    pData = (char*)(&pMsg->data[0]);
    memcpy(pData, msg, msgsz);
    retval = msgsnd(msgQueue, pMsg, msgsz, msgflg);
    free(pMsg);
  
    return retval;
}


void *SendMsgTask(void *arg)
{
    Msg_t msg;
    long tick = 0;

    if (arg)
   printf("This is task %d\n", *(int *) arg);
    else
    printf("This is task\n");
#if 1
    do {
        // send msg
        printf("Sending tick=%lu\n", tick);
        if (OK != SendMsg(&tick, sizeof(tick), 0))
        {
            LOG_ERROR("SendMsg failed");
        }
        tick++;
        usleep(500000);
        //sleep(1);
    } while (tick < 50);
    //send quit
    msg.hdr.msgType = 2;
    msgsnd(msgQueue, &msg, sizeof(msg), 0);
    return NULL;
#endif
}




int main()
{
    int retval;
    pthread_t SendMsgTaskId;
    pthread_attr_t attr;

    pthread_attr_init(&attr);
    msgQueue = CreateMsgQueue(MESSAGE1);
    if (msgQueue > 0) {
        printf("Start the thread..\n");
        retval = pthread_create(&SendMsgTaskId, &attr, SendMsgTask, NULL);
        if (OK == retval)
            pthread_join(SendMsgTaskId, NULL);
    }
    else {
        LOG_ERROR("CreateMsgQueue");
    }
    if (OK != msgctl(msgQueue, IPC_RMID, NULL))
    {
        LOG_ERROR("msgctl");
        exit(1);
    }
    pthread_attr_destroy(&attr);
    return 0;
}

Client



#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include <pthread.h>
#include <errno.h>
#include <sys/msg.h>
#include "msgdefs.h"


#define OK      0
#define LOG_ERROR(msg) { printf("%s:%u: %s %s\n", \
                __FUNCTION__, __LINE__, msg, \
                strerror(errno)); }
int msgQueue; 


int InitMsgQueue(long id)
{
    int msgId;
    key_t key;

    key = ftok("msgserv.c", id);
    msgId = msgget(key, 0644);
    if (OK != msgId)
    {
        LOG_ERROR("msgget");
    }
    return msgId;
}


int ReceiveMsg(Msg_t **pMsg, size_t *msgsz)
{
    int sz;

    int retval = 0;
    
    sz = 256;
    *pMsg = malloc(sizeof(Msg_t) + sz);
    retval = msgrcv(msgQueue, *pMsg, sz, 0, 0);
    *msgsz = sz;
    return retval;
}


void *ReceiveMsgTask(void *arg)
{
    Msg_t *pMsg;
    size_t sz;

    if (arg)
   printf("This is task %d\n", *(int *) arg);
    else
    printf("This is task\n");

    for(;;) {
        // rcv msg
        if (ReceiveMsg(&pMsg, &sz) < 0)
        {
            LOG_ERROR("ReceiveMsg");
            return NULL;
        } 
        else {
            printf("MsgType = %lu\n", pMsg->hdr.msgType);
            switch (pMsg->hdr.msgType) {
                case 1:
                    printf("Received sz=%u, tick=%lu\n", 
                            pMsg->hdr.dataSize,
                            *(long*)(&pMsg->data[0]));
                    break;
                default:
                    printf("End of task\n");
                    free(pMsg);
                    return NULL;
            }

        }
        // msg must be freed
        free(pMsg);
    } 
    return NULL;
}




int main()
{
    int retval;
    pthread_t ReceiveMsgTaskId;
    pthread_attr_t attr;

    pthread_attr_init(&attr);
    msgQueue = InitMsgQueue(MESSAGE1);
    if (msgQueue > 0) {
        printf("Start the thread..\n");
        retval = pthread_create(&ReceiveMsgTaskId, &attr, ReceiveMsgTask, NULL);
        if (OK == retval)
            pthread_join(ReceiveMsgTaskId, NULL);
    }
    else {
        LOG_ERROR("InitMsgQueue");
    }
    pthread_attr_destroy(&attr);
    return 0;
}

msgdefs.h

#ifndef __MSGDEFS_H__

#define __MSGDEFS_H__

enum {

    MESSAGE1 = 0x10000000

};

#define MSG_DATA_SIZE       1024

typedef struct {

    long msgType;

    unsigned int dataSize;

} MsgHdr_t;

typedef struct {

    MsgHdr_t hdr; /* our header */

    char data[0];   // just a place holder

} Msg_t;

#endif

Ripping VCD to Various Formats

Using VCDXRIP

To convert to MPEG:

$ vcdxrip


USING MENCODER

mencoder [options] file [file|URL|-] [-o file | file://file | smb://[user:pass@]host/filepath]

$ mencoder vcd//:2 -oac lavc -ovc lavc -o filename.avi 

($ sign is the terminal prompt)

                 -oac lavc

                      Encode with a libavcodec codec

                 -ovc lavc
                      Encode with a libavcodec codec.

Using CDFS

The source can be downloaded here:

http://users.elis.ugent.be/~mronsse/cdfs/download/

(Unfortunately, the driver doesn't yet support later Linux versions).

$ mount -t cdfs -o ro /dev/cdrom /mnt/video

And copy the *.DAT files

FFMPEG

FFMPEG is a very powerful tool (command-line) to transcode media file.

For example, to extract audio from an AVI file:

$ ffmeg -i source_video.avi -vn -ar 44100 -ac 2 192 -f mp3 sound.mp3

To convert MPG to AVI (or vise-versa):

$ ffmeg -i video_original.mpg output.avi

To encode WAV to mp3:

$ ffmpeg -i video.wav -vn -ar 44100 -ac 2 -ab 192 -f mp3 song.mp3


To mix a WAV file with a video file:

$ ffmpeg -i audio.wav -i video.avi audio_video.avi

To convert AVI to MPEG suitable for DVD:

ffmpeg -i source_video.avi -target pal-dvd -ps 2000000000 -aspect 16:9 finale_video.mpeg 

AVI to VCD:

$ ffmpeg -i video.avi -target ntsc-vcd video_out.mpg

To use 2-pass encoding, pass "-pass 2 -passlogfile somefile" before output file.

HandBrake

We then can use Handbrake to transcode video from CD/DVD to various formats, especially for Apple products.

There are two versions of HandBrake, one is the GUI front-end, and the CLI version.

To list the built-in presets:

$ HandBrakeCLI -z

To use the preset, use capital "-Z"

For example:

$HandBrakeCLI -Z "iPhone 4" -i avseq0.avi -o output.mp4

Script to convert series of WAV files

Assume there are wav files named as "disc_1.wav", "disc_2.wav" so on.
There is also a cover file (a JPEG format file) to be used as cover album embedded into each encoded MP3 file.

If we want to convert them all into MP3:



#!/bin/sh

ARTIST="Hamza Yusuf"
ALB="The Rights and Responsibilities of Marriage"
TY="2002"
GENRE="Vocal"
for i in `ls disc_*.wav | sort -V` ;
do
    TN=`echo $i | awk 'BEGIN { FPAT="[[:digit:]]+"} ; {  print $1 }'`
    lame --ti "./the_rights_and_responsibilities_of_marriage.jpg" --tt "$ALB cd$TN" --ta "$ARTIST" --tl i"$ALB" --ty "$TY" --tn "$TN/13" --tg "$GENRE" $i "$ALB cd${TN}.mp3"

done

Youtube Video to MP3

1. Download script youtube-dl from  http://rg3.github.com/youtube-dl/
2. Make sure we have ffmpeg installed
3. Write a small script with content as below:

youtube-dl --extract-audio --audio-format mp3 -v -t "$1"

For example, to download and extract the audio and save it as MP3 (assuming the small script above has been saved as utube":

utube http://www.youtube.com/watch?v=RoqRbe3dgwU&feature=g-vrec

Post-processing:

concatenate multiple MP3 files to become one mp3:

mp3wrap <file1> <file2> ... <output>

To downmix the result file to a mono-aural:

lame -m s -a <file> <downmix.mp3>


to copy the ID3 data from the original file to the output file

id3cp <mp3 source> <mp3 dest>

How to set root password on ReadyNAS

Recently I had forgotten my ReadyNAS password.  Even worse, I couldn't login at all due reckless upgrade I did on the box.  I was able to undo the upgrade by letting the box recopy the Linux from its firmware to the drive (see ReadyNAS forum/documentation on how to make the ReadyNAS box recopy the firmware), but still I forgot what my root password was.

Various steps I had tried as I found on the Internet as well as on the ReadyNAS website, but none of them work. Finally, I had an idea to just access the drive directly thru SATA-to-USB cable and reset the password manually.

Basically what I did was to set the root password stored in the file /etc/passwd (fortunately the authententication is still the old-fashioned one, where the MD5 encrypted password stored directly in the file instead in shadow file).

You might ask, "How the hell I access the drive?".  Well, first you need remove the drive from the ReadyNAS bay (make sure it is turned off!!) then  attach the SATA-to-USB cable to the drive.  Connect the usb end to our PC.

We cannot mount directly to the drive, because some other issues.  To mount, see my previous post ("How to mount disk used by ReadyNAS").  Once it is  mounted (just mount the ext3 partition [first partition], no need to mount the LVM), we can now modify the file /etc/passwd.

First, save the following script (thanks to somebody who posted it on the Internet), say, as /home/<yourloginname>/bin/setpasswd:


#!/usr/bin/perl
################################################################################
# Generate an MD5 hash for a string.
# Created to allow me to set a blank Linux password. Required this to create
# multiple VsFTP accounts with anonymous style credientials.
#
# If all you want is the MD5 Hash for NULL (blank password) here's one...
# $1$VNMbpxGH$sew7cnwH9ixU.x27UbFNn.
#
# Advice: If replacing a Linux password with a blank string, ensure you give 
# the user a shell of /sbin/nologin as you wouldn't want them to login!
################################################################################
# Load dependancies...
# perl -MCPAN -e 'install Crypt::PasswdMD5'
################################################################################

use strict;
use Crypt::PasswdMD5 qw(unix_md5_crypt);
my @salt = ( '.', '/', 0 .. 9, 'A' .. 'Z', 'a' .. 'z' );
my %encrypted;


sub abort {
print "ABORT: $_[0]\n";
exit 1
}


sub gensalt { #------------------------------------------------------------
# uses global @salt to construct salt string of requested length
my $count = shift;

my $salt;
for (1..$count) {
$salt .= (@salt)[rand @salt];
}

return $salt;
} # end gensalt


sub get_encryptedpw { #--------------------------------------------------
my $unencrypted="$_[0]";

# generate traditional (weak!) DES password, and more modern md5
$encrypted{des} = crypt( $unencrypted, gensalt(2) );
$encrypted{md5} = unix_md5_crypt( $unencrypted, gensalt(8) );

return %encrypted;
}

################################################################################
print "Enter password string to encrypt (can be blank) : ";
my $password = <STDIN>;
chomp $password;

get_encryptedpw($password);

print "Plaintext \"$password\" = MD5 Hash: $encrypted{md5}\n";
print "\nReplace the /etc/shadow password string with the above to force pass change\n";


(Don't forget to make it executable by doing "chmod +x ./setpasswd)
Run the script.  It will ask you to enter a password.  An example of the output (with blank password):


$ setpasswd
Enter password string to encrypt (can be blank) : 
Plaintext "" = MD5 Hash: $1$udf2EDLY$a/cLQQ4h25rwZQc9VKmG6/

Replace the /etc/shadow password string with the above to force pass change

Copy the portion after the string "MD5 Hash: " above and paste it in the <READYNAS MOUNTPOINT>/etc/passwd.  To be precise, it should be pasted in line where "root:...". Paste it right after "root:", and let the rest of the existing text still intact.

Save the file, unmount (just use the regular umount for this one), reinsert the drive into the ReadyNAS bay and turn it on.  Once it is running, try to SSH to the box as root and enter the new password.  It should work!