电商商品概念与销售知识图谱

GoodsKG

　　GoodsKG, 基于京东网站的1300种商品上下级概念，约10万商品品牌，约65万品牌销售关系，商品描述维度等知识库，基于该知识库可以支持商品属性库构建，商品销售问答，品牌物品生产等知识查询服务，也可用于情感分析等下游应用．

项目介绍

　　概念层级知识是整个常识知识体系中的重要组成部分。概念层级目前包括百科性概念层级和专业性概念层级两类，百度百科概念体系以及互动百科分类体系是其中的一个代表。就后者而言，目前出现了许多垂直行业概念层级，如医疗领域概念体系，垫上领域概念体系，其中以淘宝、京东等为代表的电商网站在以商品为中心上，构建起了一种商品概念目录层级以及商品与品牌，品牌与品牌之间的关联关系。
　　本项目认为，电商网站中的商品分类目录能够供我们构建起一个商品概念体系，基于商品首页，我们可以得到商品与商品品牌之间的关系，商品的属性以及属性的取值信息。基于这类信息，又可以进一步得到商品的画像以及商品品牌的画像。基于该画像。可以对自然语言处理处理的几个下游应用带来帮助，如商品品牌识别，商品对象及属性级别情感分析，商品评价短语库构建，商品品牌竞争关系梳理等。
　　因此，本项目以京东电商为实验数据来源，采集京东商品目录树，并获取其对应的底层商品概念信息，组织形成商品知识图谱。目前，该图谱包括有概念的上下位is a关系以及商品品牌与商品之间的销售sale关系共两类关系，涉及商品概念数目1300+，商品品牌数目约10万+，属性数目几千种，关系数目65万规模。该项目可以进一步增强商品领域概念体系的应用，对自然语言处理处理的几个下游应用带来帮助，如商品品牌识别，商品对象及属性级别情感分析，商品评价短语库构建，商品品牌竞争关系梳理等提供基础性的概念服务。

数据介绍

1, 基本数据内容 

 

2, is-a概念上下位关系 

MATCH p=()-[r:is_a]->() RETURN p LIMIT 25

3, sale销售关系

MATCH p=()-[r:sales]->() RETURN p LIMIT 25

4, 混合关联关系 

MATCH (n:Product) where n.name=’手机’ RETURN n LIMIT 25

GitHub源码

https://github.com/chenjj9527/ProductKnowledgeGraph-master

#!/usr/bin/env python3
# coding: utf-8
# File: build_kg.py
# Author: cjj
# Date: 19-12-23

import urllib.request
from urllib.parse import quote_plus
from lxml import etree
import gzip
import chardet
import json
import pymongo

class GoodSchema:
    def __init__(self):
        self.conn = pymongo.MongoClient()
        return

    '''获取搜索页'''
    def get_html(self, url):
        headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/600.5.17 (KHTML, like Gecko) Version/8.0.5 Safari/600.5.17"}
        try:
            req = urllib.request.Request(url, headers=headers)
            data = urllib.request.urlopen(req).read()
            coding = chardet.detect(data)
            html = data.decode(coding['encoding'])
        except:
            req = urllib.request.Request(url, headers=headers)
            data = urllib.request.urlopen(req).read()
            html = data.decode('gbk')


        return html

    '''获取详情页'''
    def get_detail_html(self, url):
        headers = {
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "en-US,en;q=0.9",
            "cache-control": "max-age=0",
            "referer": "https://www.jd.com/allSort.aspx",
            "upgrade-insecure-requests": 1,
            "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/66.0.3359.181 Chrome/66.0.3359.181 Safari/537.36"
        }
        try:
            req = urllib.request.Request(url, headers=headers)
            data = urllib.request.urlopen(req).read()
            html = gzip.decompress(data)
            coding = chardet.detect(html)
            html = html.decode(coding['encoding'])
        except Exception as e:
            req = urllib.request.Request(url, headers=headers)
            data = urllib.request.urlopen(req).read()
            html = gzip.decompress(data)
            html = html.decode('gbk')
        return html


    '''根据主页获取数据'''
    def home_list(self):
        url = 'https://www.jd.com/allSort.aspx'
        html = self.get_html(url)
        selector = etree.HTML(html)
        divs = selector.xpath('//div[@class= "category-item m"]')
        for indx, div in enumerate(divs):
            first_name = div.xpath('./div[@class="mt"]/h2/span/text()')[0]
            second_classes = div.xpath('./div[@class="mc"]/div[@class="items"]/dl')
            for dl in second_classes:
                second_name = dl.xpath('./dt/a/text()')[0]
                third_classes = ['https:' + i for i in dl.xpath('./dd/a/@href')]
                third_names = dl.xpath('./dd/a/text()')
                for third_name, url in zip(third_names, third_classes):
                    try:
                        attr_dict = self.parser_goods(url)
                        attr_brand = self.collect_brands(url)
                        attr_dict.update(attr_brand)
                        data = {}
                        data['fisrt_class'] = first_name
                        data['second_class'] = second_name
                        data['third_class'] = third_name
                        data['attrs'] = attr_dict
                        self.conn['goodskg']['data'].insert(data)
                        print(indx, len(divs), first_name, second_name, third_name)
                    except Exception as e:
                        print(e)
        return

    '''解析商品数据'''
    def parser_goods(self, url):
        html = self.get_detail_html(url)
        selector = etree.HTML(html)
        title = selector.xpath('//title/text()')
        attr_dict = {}
        other_attrs = ''.join([i for i in html.split('
') if 'other_exts' in i])
        other_attr = other_attrs.split('other_exts =[')[-1].split('];')[0]
        if other_attr and 'var other_exts ={};' not in other_attr:
            for attr in other_attr.split('},'):
                if '}' not in attr:
                    attr = attr + '}'
                data = json.loads(attr)
                key = data['name']
                value = data['value_name']
                attr_dict[key] = value
        attr_divs = selector.xpath('//div[@class="sl-wrap"]')
        for div in attr_divs:
            attr_name = div.xpath('./div[@class="sl-key"]/span/text()')[0].replace('：','')
            attr_value = ';'.join([i.replace('  ','') for i in div.xpath('./div[@class="sl-value"]/div/ul/li/a/text()')])
            attr_dict[attr_name] = attr_value

        return attr_dict

    '''解析品牌数据'''
    def collect_brands(self, url):
        attr_dict = {}
        brand_url = url + '&sort=sort_rank_asc&trans=1&md=1&my=list_brand'
        html = self.get_html(brand_url)
        if 'html' in html:
            return attr_dict
        data = json.loads(html)
        brands = []

        if 'brands' in data and data['brands'] is not None:
            brands = [i['name'] for i in data['brands']]
        attr_dict['品牌'] = ';'.join(brands)

        return attr_dict



if __name__ == '__main__':
    handler = GoodSchema()
    handler.home_list()

#!/usr/bin/env python3
# coding: utf-8
# File: build_kg.py
# Author: cjj
# Date: 19-12-23

import json
import os
from py2neo import Graph, Node, Relationship


class GoodsKg:
    def __init__(self):
        cur = '/'.join(os.path.abspath(__file__).split('/')[:-1])
        self.data_path = os.path.join(cur, 'data/goods_info.json')
        self.g = Graph(
            host="127.0.0.1",  # neo4j 搭载服务器的ip地址，ifconfig可获取到
            http_port=7474,  # neo4j 服务器监听的端口号
            user="neo4j",  # 数据库user name，如果没有更改过，应该是neo4j
            password="111111")
        return

    '''读取数据'''
    def read_data(self):
        rels_goods = []
        rels_brand = []
        goods_attrdict = {}
        concept_goods = set()
        concept_brand = set()
        count = 0
        for line in open(self.data_path,encoding='UTF-8'):
            count += 1
            print(count)
            line = line.strip()
            data = json.loads(line)
            first_class = data['fisrt_class'].replace("'",'')
            second_class = data['second_class'].replace("'",'')
            third_class = data['third_class'].replace("'",'')
            attr = data['attrs']
            concept_goods.add(first_class)
            concept_goods.add(second_class)
            concept_goods.add(third_class)
            rels_goods.append('@'.join([second_class, 'is_a', '属于', first_class]))
            rels_goods.append('@'.join([third_class, 'is_a', '属于', second_class]))

            if attr and '品牌' in attr:
                brands = attr['品牌'].split(';')
                for brand in brands:
                    brand = brand.replace("'",'')
                    concept_brand.add(brand)
                    rels_brand.append('@'.join([brand, 'sales', '销售', third_class]))

            goods_attrdict[third_class] = {name:value for name,value in attr.items() if name != '品牌'}

        return concept_brand, concept_goods, rels_goods, rels_brand

    '''构建图谱'''
    def create_graph(self):
        concept_brand, concept_goods, rels_goods, rels_brand = self.read_data()
        print('creating nodes....')
        self.create_node('Product', concept_goods)
        self.create_node('Brand', concept_brand)
        print('creating edges....')
        self.create_edges(rels_goods, 'Product', 'Product')
        self.create_edges(rels_brand, 'Brand', 'Product')
        return

    '''批量建立节点'''
    def create_node(self, label, nodes):
        pairs = []
        bulk_size = 1000
        batch = 0
        bulk = 0
        batch_all = len(nodes)//bulk_size
        print(batch_all)
        for node_name in nodes:
            sql = """CREATE(:%s {name:'%s'})""" % (label, node_name)
            pairs.append(sql)
            bulk += 1
            if bulk % bulk_size == 0 or bulk == batch_all+1:
                sqls = '
'.join(pairs)
                self.g.run(sqls)
                batch += 1
                print(batch*bulk_size,'/', len(nodes), 'finished')
                pairs = []
        return


    '''构造图谱关系边'''
    def create_edges(self, rels, start_type, end_type):
        batch = 0
        count = 0
        for rel in set(rels):
            count += 1
            rel = rel.split('@')
            start_name = rel[0]
            end_name = rel[3]
            rel_type = rel[1]
            rel_name = rel[2]
            sql = 'match (m:%s), (n:%s) where m.name = "%s" and n.name = "%s" create (m)-[:%s{name:"%s"}]->(n)' %(start_type, end_type, start_name, end_name,rel_type,rel_name)
            try:
                self.g.run(sql)
            except Exception as e:
                print(e)
            if count%10 == 0:
                print(count)

        return


if __name__ =='__main__':
    handler = GoodsKg()
    handler.create_graph()