纯真IP地址数据库qqwry.dat解析

ip地址数据库，在现在互联网时代非常有用，比如大型网站的用户安全保护系统，就常常会根据ip反查的信息，甄别账号的一些不安全登录行为，比如跨区域登录问题等。ip其实关联了一些有信息，比如区域，所在运营商，一些收录全的，甚至包括具体经纬度，像百度的IP定位api就比较全。下面来介绍一下“ 纯真IP地址数据库qqwry”的格式以及解析

以下是“ 纯真IP地址数据库qqwry”官网对其的介绍。

纯真版IP地址数据库是当前网络上最权威、地址最精确、IP记录以及网吧数据最多的IP地址数据库。收集了包括中国电信、中国移动、中国联通、铁通、长城宽带等各 ISP 的最新准确 IP 地址数据。通过大家的共同努力打造一个没有未知数据，没有错误数据的QQ IP。IP数据库每5天更新一次，请大家定期更新最新的IP数据库！

格式

+———-+
| 文件头 | (8字节)
+———-+
| 记录区 | (不定长)
+———-+
| 索引区 | (大小由文件头决定)
+———-+

使用java语言解析的两种思路：

• 使用内存映射文件方式读取，使用java的MappedByteBuffer 将原数据文件映射到MappedByteBuffer对象中，然后通过MappedByteBuffer 提供的字节读取方式实现ip的查找。搜索是在索引区使用二分法

• 使用byte数组读取，及将二进制的数据库信息全都按顺序读入到一个数组中，由于数据是有格式的，我们便可计算根据索引区和记录区在数组中的位置，当查询ip时，从数组中的索引区开始通过二分查找方式找到IP地址对应的国家和区域的位置，然后从数组中取出地区信息。

热升级思路：

使用一个可调度的单线程的线程池，线程定时检测qqwry.dat文件是否修改，若修改则重新将数据重新载入，载入过程可使用可重入锁ReentrantLock来锁住资源，避免在更新的过程中脏查询

两种解析方式的实现源码如下：
方式一（MappedByteBuffer ）：

package com.difeng.qqwry1;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/**
 * @Description:ip定位查找工具(使用内存映射文件方式读取，线程安全)
 * @author:difeng
 * @date:2016年12月11日
 */
public class IPLocation {

	private static final int IP_RECORD_LENGTH = 7;

	private static final byte REDIRECT_MODE_1 = 0x01;

	private static final byte REDIRECT_MODE_2 = 0x02;

	private MappedByteBuffer mbbFile;

	private static Long lastModifyTime = 0L;

	public static boolean enableFileWatch = false;

	private static ReentrantLock lock = new ReentrantLock();

	private File qqwryFile;

	private long firstIndexOffset;

	private long lastIndexOffset;

	private long totalIndexCount;

	public IPLocation(String filePath) throws Exception {
		this.qqwryFile = new File(filePath);
		load();
		if (enableFileWatch) {
			watch();
		}
	}

	private void watch(){
		Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
			@Override
			public void run() {
				long time = qqwryFile.lastModified();
				if (time > lastModifyTime) {
					lastModifyTime = time;
					try {
						load();
					} catch (Exception e) {
						e.printStackTrace();
					}
				}
			}
		}, 1000L, 30000L, TimeUnit.MILLISECONDS);
	}

	public long read4ByteAsLong(long pos) {
		mbbFile.position((int)pos);
		return 0xFFFFFFFFL & mbbFile.getInt();
	}

	public long read3ByteAsLong(long pos){
		mbbFile.position((int)pos);
		return 0xFFFFFFL & mbbFile.getInt();
	}


	@SuppressWarnings("resource")
	private void load() throws Exception {
		lastModifyTime = qqwryFile.lastModified();
		lock.lock();
		try {
			mbbFile =  new RandomAccessFile(qqwryFile, "r")
					.getChannel()
					.map(FileChannel.MapMode.READ_ONLY, 0, qqwryFile.length());
			mbbFile.order(ByteOrder.LITTLE_ENDIAN);
			firstIndexOffset = read4ByteAsLong(0);
			lastIndexOffset = read4ByteAsLong(4);
			totalIndexCount = (lastIndexOffset - firstIndexOffset) / IP_RECORD_LENGTH + 1;
		} finally {
			lock.unlock();
		}
	}
	
	/**
	 * @Description:将“.”号分隔的字符串转换为long类型的数字，字节序例如:
	 *   ip:182.92.240.48  16进制表示(B6.5C.F0.30)
	 *   转换后ip的16进制表示:0xB65CF030
	 * @param ipStr
	 * @return:long
	 */
	private static long inet_pton(String ipStr) {
		if(ipStr == null){
			throw new NullPointerException("ip不能为空");
		}
		String [] arr = ipStr.split("\.");
		long ip = (Long.parseLong(arr[0])  & 0xFFL) << 24 & 0xFF000000L;
		ip |=  (Long.parseLong(arr[1])  & 0xFFL) << 16 & 0xFF0000L;
		ip |=  (Long.parseLong(arr[2])  & 0xFFL) << 8 & 0xFF00L;
		ip |=  (Long.parseLong(arr[3])  & 0xFFL);
		return ip;
	}

	private long search(long ip) {
		long low = 0;
		long high = totalIndexCount;
		long mid = 0;
		while(low <= high) {
			mid = (low + high) >>> 1 ;
		    long indexIP = read4ByteAsLong(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH);
		    long nextIndexIP =  read4ByteAsLong(firstIndexOffset + mid * IP_RECORD_LENGTH);
		    if(indexIP <= ip && ip < nextIndexIP) {
		    	return read3ByteAsLong(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH + 4);
		    } else {
		    	if(ip > indexIP) {
		    		low = mid + 1;
		    	} else if(ip < indexIP) {
		    		high = mid - 1;
		    	}
		    }
		}
		return -1;
	}

	private Location readIPLocation(long offset) {
		try {
			mbbFile.position((int)offset + 4);
			Location loc = new Location();
			byte redirectMode = mbbFile.get();
			if (redirectMode == REDIRECT_MODE_1) {
				long countryOffset = read3ByteAsLong((int)offset + 5);
				mbbFile.position((int)countryOffset);
				redirectMode = mbbFile.get();
				if (redirectMode == REDIRECT_MODE_2) {
					loc.country = readString(read3ByteAsLong(countryOffset + 1));
					mbbFile.position((int)countryOffset + 4);
				} else {
					loc.country = readString(countryOffset);
				}
				loc.area = readArea(mbbFile.position());
			} else if (redirectMode == REDIRECT_MODE_2) {
				loc.country = readString(read3ByteAsLong((int)offset + 5));
				loc.area = readArea((int)offset + 8);
			} else {
				loc.country = readString(mbbFile.position() - 1);
				loc.area = readArea(mbbFile.position());
			}
			return loc;
		} catch (Exception e) {
			return null;
		}
	}

	private String readArea(int offset) {
		mbbFile.position(offset);
		byte redirectMode = mbbFile.get();
		if (redirectMode == REDIRECT_MODE_1 || redirectMode == REDIRECT_MODE_2) {
			long areaOffset = read3ByteAsLong((int)offset + 1);
			if (areaOffset == 0){
				return "";
			} else {
				return readString(areaOffset);
			}
		} else {
			return readString(offset);
		}
	}

	private String readString(long offset) {
		try {
			mbbFile.position((int)offset);
			byte[] buf = new byte[128];
			int i;
			for (i = 0, buf[i] = mbbFile.get(); buf[i] != 0; buf[++i] = mbbFile.get());
			
			if (i != 0){
			    return new String(buf, 0, i, "GBK");
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		return "";
	}

	public  Location fetchIPLocation(String ip) {
		lock.lock();
		try {
			long offset = search(inet_pton(ip));
			if(offset != -1){
				return readIPLocation(offset);
			}
		} finally {
			lock.unlock();
		}
		return null;
	}
}

方式二（数组方式）：

package com.difeng.qqwry2;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/**
 * @Description:ip定位(使用byte数据方式读取)
 * @author:difeng
 * @date:2016年12月13日
 */
public class IPLocation {
	
	private  byte[] data;
	
	private  long firstIndexOffset;
	
	private  long lastIndexOffset;
	
	private  long totalIndexCount;
	
	private static final byte REDIRECT_MODE_1 = 0x01;
	
	private static final byte REDIRECT_MODE_2 = 0x02;
	
	static   final long IP_RECORD_LENGTH = 7;
	
	private static ReentrantLock lock = new ReentrantLock();
	
	private static Long lastModifyTime = 0L;

	public static boolean enableFileWatch = false;
	
	private File qqwryFile;
	
	public IPLocation(String  filePath) throws Exception {
		this.qqwryFile = new File(filePath);
		load();
		if(enableFileWatch){
			watch();
		}
	}
	
    private void watch() {
    	Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
			@Override
			public void run() {
				long time = qqwryFile.lastModified();
				if (time > lastModifyTime) {
					lastModifyTime = time;
					try {
						load();
						System.out.println("reload");
					} catch (Exception e) {
						e.printStackTrace();
					}
				}
			}
		}, 1000L, 5000L, TimeUnit.MILLISECONDS);
    }
    
	private void load() throws Exception {
		lastModifyTime = qqwryFile.lastModified();
		ByteArrayOutputStream out = null;
		FileInputStream in = null;
		lock.lock();
		try {
			out = new ByteArrayOutputStream();
			byte[] b = new byte[1024];
			in = new FileInputStream(qqwryFile);
			while(in.read(b) != -1){
				out.write(b);
			}
			data = out.toByteArray();
			firstIndexOffset = read4ByteAsLong(0);
			lastIndexOffset = read4ByteAsLong(4);
			totalIndexCount = (lastIndexOffset - firstIndexOffset) / IP_RECORD_LENGTH + 1;
			in.close();
			out.close();
		} finally {
			try {
				if(out != null) {
					out.close();
				}
				if(in != null) {
					in.close();
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
			lock.unlock();
		}
	}
	
	private long read4ByteAsLong(final int  offset) {
		long val = data[offset] & 0xFF;
		val |= (data[offset + 1] << 8L) & 0xFF00L;
		val |= (data[offset + 2] << 16L) & 0xFF0000L;
		val |= (data[offset + 3] << 24L) & 0xFF000000L;
		return val;
	}

	private long read3ByteAsLong(final int offset) {
		long val = data[offset] & 0xFF;
		val |= (data[offset + 1] << 8) & 0xFF00;
		val |= (data[offset + 2] << 16) & 0xFF0000;
		return val;
	}
    
	private long search(long ip) {
		long low = 0;
		long high = totalIndexCount;
		long mid = 0;
		while(low <= high){
			mid = (low + high) >>> 1 ;
		    long indexIP = read4ByteAsLong((int)(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH));
	        long indexIPNext = read4ByteAsLong((int)(firstIndexOffset + mid * IP_RECORD_LENGTH));
		    if(indexIP <= ip && ip < indexIPNext) {
		    	return read3ByteAsLong((int)(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH + 4));
		    } else {
		    	if(ip > indexIP) {
				    low = mid + 1;
				} else if (ip < indexIP) {
				    high = mid - 1;
				}
		    }
		}
		return -1;
	}
	
	public Location fetchIPLocation(String ip) {
		long numericIp = inet_pton(ip);
		lock.lock();
		long offset = search(numericIp);
		try{
			if(offset != -1) {
				return readIPLocation((int)offset);
			}
		} finally {
		    lock.unlock();
		}
		return null;
	}
	
	private Location readIPLocation(final int offset) {
		final Location loc = new Location();
		try {
			byte redirectMode = data[offset + 4];
			if (redirectMode == REDIRECT_MODE_1) {
				long countryOffset = read3ByteAsLong((int)offset + 5);
				redirectMode = data[(int)countryOffset];
				if (redirectMode == REDIRECT_MODE_2) {
					final QQwryString country = readString((int)read3ByteAsLong((int)countryOffset + 1));
					loc.country = country.string;
					countryOffset = countryOffset + 4;
				} else {
					final QQwryString country = readString((int)countryOffset);
					loc.country = country.string;
					countryOffset += country.byteCountWithEnd;
				}
				loc.area = readArea((int)countryOffset);
			} else if (redirectMode == REDIRECT_MODE_2) {
				loc.country = readString((int)read3ByteAsLong((int)offset + 5)).string;
				loc.area = readArea((int)offset + 8);
			} else {
				final QQwryString country = readString((int)offset + 4);
				loc.country = country.string;
				loc.area = readArea((int)offset + 4 + country.byteCountWithEnd);
			}
			return loc;
		} catch (Exception e) {
			return null;
		}
	}

	private String readArea(final int offset) {
		byte redirectMode = data[offset];
		if (redirectMode == REDIRECT_MODE_1 || redirectMode == REDIRECT_MODE_2) {
			long areaOffset = read3ByteAsLong((int)offset + 1);
			if (areaOffset == 0) {
				return "";
			} else {
				return readString((int)areaOffset).string;
			}
		} else {
			return readString(offset).string;
		}
	}
	
	private QQwryString readString(int offset) {
		int pos = offset;
		final byte[] b = new byte[128];
		int i;
		for (i = 0, b[i] = data[pos++]; b[i] != 0; b[++i] = data[pos++]);
		try{
			   return new QQwryString(new String(b,0,i,"GBK"),i + 1);
		} catch(UnsupportedEncodingException e) {
			return new QQwryString("",0);
		}
	}
	
	 /**
     * @Description:“.”号分隔的字符串转换为long类型的数字
     * @param ipStr 
     * @return:long
     */
	private static long inet_pton(String ipStr) {
		if(ipStr == null){
			throw new NullPointerException("ip不能为空");
		}
		String [] arr = ipStr.split("\.");
		long ip = (Long.parseLong(arr[0])  & 0xFFL) << 24 & 0xFF000000L;
		ip |=  (Long.parseLong(arr[1])  & 0xFFL) << 16 & 0xFF0000L;
		ip |=  (Long.parseLong(arr[2])  & 0xFFL) << 8 & 0xFF00L;
		ip |=  (Long.parseLong(arr[3])  & 0xFFL);
		return ip;
	}
	
	private class QQwryString{
		
		public final String string;
		
		public final int byteCountWithEnd;
		
		public QQwryString(final String string,final int byteCountWithEnd) {
			this.string = string;
			this.byteCountWithEnd = byteCountWithEnd;
		}
		
		@Override
		public String toString() {
			return string;
		}
		
	}
}

以上为主要代码，获取全部代码请点击全部代码

使用

final IPLocation ipLocation = new IPLocation(filePath);
Location loc = ipl.fetchIPLocation("182.92.240.50");
System.out.printf("%s %s",loc.country,loc.area);

格式改进

由于原格式中读取地区记录时采用重定向，有些繁琐。去掉之后格式更简单，国家和地区单独存放,索引里分别记录的国家和地区的地址。
新格式如下：

+----------+
| 文件头 | (8字节)
+----------+
| 记录区 | (不定长)
+----------+
| 索引区 | (大小由文件头决定)
+----------+
文件头：
+------------------------------+-----------------------------+
| first index position(4 bytes)|last index position(4 bytes) |
+------------------------------+-----------------------------+
记录区：
+------------------+----------+------------------+----------+-----
| country1(n bytes)|(1 byte)| country2(n bytes)|(1 byte)|...
+------------------+----------+------------------+----------+-----
+------------------+----------+------------------+----------+-----
| area1(n bytes) |(1 byte)| area2(n bytes) |(1 byte)|...
+------------------+----------+------------------+----------+-----
索引区：
+------------+-------------------------+------------------------+
|ip1(4 bytes)|country position(3 bytes)| area position(3 bytes) |...
+------------+-------------------------+------------------------+
转换方法：

final IPFileConvertor convertor = new 
IPFileConvertor(IPFileConvertor.class.getResource("/qqwry.dat").getPath(),"./qqwry.dat");
convertor.convert();
新格式使用方法和之前的一致，使用com.difeng.convert包下的解析类IPLocation解析即可。

相关连接：
qqwry下载： qqwry
全球ip地址库（收费）：IPLocation