基于大数据的房价数据可视化分析预测系统

base_url = 'https://baoding.xxxx.com/community/p{}/'

all_xqlb_links = set()
for page in range(1, 51):
    url = base_url.format(page)
    # 获取 html 页码，并进行dom解析
    # ...

def get_house_info(house_link):
    """获取房屋的信息"""
    headers = {
        'accept': '*/*',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'cookie': 'Your cookie',
        'referer': house_link,
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'same-origin',
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36'
    }

    response = requests.get(house_link, headers=headers)
    response.encoding = 'utf8'
    soup = BeautifulSoup(response.text, 'lxml')
    
    省略其他代码
    
    total_price = soup.select('span.maininfo-price-num')[0]
    total_price = total_price.text.strip()
    house_info['总价'] = total_price
    

    。。。。。。

    
    tags = soup.select('div.maininfo-tags')[0].text
    if '电梯' in tags:
        house_info['配套电梯'] = '有'
    
    # 所属区域
    for line in soup.select('div.maininfo-community-item'):
        text = line.text.strip()
        if '所属区域' in text:
            house_info['所在位置'] = text.split(' ')[1].split('\xa0')[0]
            
    return house_info

for house_info in all_house_infos:
    for key in all_keys:
        if key not in house_info:
            house_info[key] = '暂无'
    
    if isinstance(house_info['单价'], float):
        continue
        
    house_info['单价'] = float(house_info['单价'][:-3].strip())
    house_info['总价'] = float(house_info['总价'].strip())
    

    省略其他代码

    
    if '(' not in house_info['所在楼层']:
        house_info['所在楼层'] = '底层({})'.format(house_info['所在楼层'])
    
    house_info['总楼层'] = list(map(int, re.findall(r'(\d+)', house_info['所在楼层'])))[0]
    house_info['所在楼层'] = house_info['所在楼层'][:2]
    
    

print('---> cv train to choose best_num_boost_round')
dtrain = xgb.DMatrix(train_X, label=train_Y, feature_names=df_columns)

xgb_params = {
    'learning_rate': 0.005,
    'n_estimators': 4000,
    'max_depth': 3,
    'min_child_weight': 1.5,
    'eval_metric': 'rmse',
    'objective': 'reg:linear',
    'nthread': -1,
    'silent': 1,
    'booster': 'gbtree'
}

cv_result = xgb.cv(dict(xgb_params),
                   dtrain,
                   num_boost_round=4000,
                   early_stopping_rounds=100,
                   verbose_eval=400,
                   show_stdv=False,
                   )
best_num_boost_rounds = len(cv_result)
mean_train_logloss = cv_result.loc[best_num_boost_rounds-11 : best_num_boost_rounds-1, 'train-rmse-mean'].mean()
mean_test_logloss = cv_result.loc[best_num_boost_rounds-11 : best_num_boost_rounds-1, 'test-rmse-mean'].mean()
print('best_num_boost_rounds = {}'.format(best_num_boost_rounds))

print('mean_train_rmse = {:.7f} , mean_valid_rmse = {:.7f}\n'.format(mean_train_logloss, mean_test_logloss))

---> cv train to choose best_num_boost_round
[0]	train-rmse:4.10205	test-rmse:4.10205
[400]	train-rmse:0.59919	test-rmse:0.605451
[800]	train-rmse:0.20857	test-rmse:0.230669
[1200]	train-rmse:0.185981	test-rmse:0.21354
[1600]	train-rmse:0.181188	test-rmse:0.211841
[2000]	train-rmse:0.177933	test-rmse:0.211291
[2400]	train-rmse:0.174346	test-rmse:0.210886
best_num_boost_rounds = 2512
mean_train_rmse = 0.1733781 , mean_valid_rmse = 0.2108875

print('决策树模型在验证集上的均方误差 RMSE 为：', rmse(valid_Y, predict_valid))

>> 决策树模型在验证集上的均方误差 RMSE 为： 0.19991482173207226