#!/usr/bin/env python

-- coding: utf-8 --

"""Description

"""

import torch

import numpy as np

from ptranking.ltr_global import ltr_seed
from ptranking.ltr_cegan.ltr_ceadversarial import AdLTREvaluator
from ptranking.ltr_adversarial.eval.ltr_adversarial import AdLTREvaluator
np.random.seed(seed=ltr_seed)
torch.manual_seed(seed=ltr_seed)

if name == 'main':

"""
>>> Learning-to-Rank Models <<< 

(2) Adversarial Optimization
-----------------------------------------------------------------------------------------
| Pointwise | IRGAN_Point % IRGAN_PointCE                                                |
-----------------------------------------------------------------------------------------
| Pairwise  | IRGAN_Pair % IRGAN_PairCE                                                  |
-----------------------------------------------------------------------------------------
| Listwise  | IRGAN_List % IRGAN_ListCE                                                  |
-----------------------------------------------------------------------------------------


>>> Supported Datasets <<<
-----------------------------------------------------------------------------------------
| LETTOR    | MQ2007_Super %  MQ2008_Super %  MQ2007_Semi %  MQ2008_Semi                |
-----------------------------------------------------------------------------------------
| MSLRWEB   | MSLRWEB10K %  MSLRWEB30K                                                  |
-----------------------------------------------------------------------------------------
| Yahoo_LTR | Set1 % Set2                                                               |
-----------------------------------------------------------------------------------------
| ISTELLA_LTR | Istella_S | Istella | Istella_X                                         |
-----------------------------------------------------------------------------------------
| IRGAN_MQ2008_Semi                                                                      |
-----------------------------------------------------------------------------------------

"""

cuda = 0  # the gpu id, e.g., 0 or 1, otherwise, set it as None indicating to use cpu

debug = False  # in a debug mode, we just check whether the model can operate
config_with_json = False  # specify configuration with json files or not

models_to_run = [
    # 'IRGAN_Point',
     'IRGAN_Pair'
    # 'IRGAN_List',

    # 'IRGAN_PointCE',

    # 'IRGAN_ListCE'
]
evaluator = AdLTREvaluator(cuda=cuda)

if config_with_json:  # specify configuration with json files
    # the directory of json files
    dir_json = '/ltr_adversarial/json/'
    # dir_json = '/home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adversarial/json/'
    for model_id in models_to_run:
        evaluator.run(debug=debug, model_id=model_id, config_with_json=config_with_json, dir_json=dir_json)

else:  # specify configuration manually
    data_id = 'MQ2008_Semi'
   # data_id = 'OHSUMED'

    ''' pointsf | listsf, namely the type of neural scoring function '''
    sf_id = 'pointsf'

    ''' Location of the adopted data '''
    # dir_data = '/home/dl-box/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/'
    # dir_data = '/Users/solar/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/'
    dir_data = 'dataset/MQ2008-semi/'
   # dir_data = 'dataset/OHSUMED/'

    ''' Output directory '''
    # dir_output = '/Users/dryuhaitao/WorkBench/CodeBench/Bench_Output/NeuralLTR/ALTR/'
    # dir_output = '/home/dl-box/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/Listwise/'
    # dir_output = '/Users/solar/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/'
    dir_output = 'Output/NeuralLTR/'

    grid_search = False  # with grid_search, we can explore the effects of different hyper-parameters of a model

    for model_id in models_to_run:
        evaluator.run(debug=debug,train=True,model_id=model_id, sf_id=sf_id,
                      data_id=data_id, dir_data=dir_data, dir_output=dir_output, grid_search=grid_search)

Fold- 1
loading buffered file ...
loading buffered file ...
loading buffered file ...
2 G - nDCG@k - [0.6165324]
2 D - nDCG@k - [0.19242235]
3 G - nDCG@k - [0.6165324]
3 D - nDCG@k - [0.19242235]
4 G - nDCG@k - [0.6165324]
4 D - nDCG@k - [0.19242235]
5 G - nDCG@k - [0.6165324]
5 D - nDCG@k - [0.19242235]
6 G - nDCG@k - [0.6165324]
6 D - nDCG@k - [0.19242235]
7 G - nDCG@k - [0.6165324]
7 D - nDCG@k - [0.19242235]
8 G - nDCG@k - [0.6165324]
8 D - nDCG@k - [0.19242235]
9 G - nDCG@k - [0.6165324]
9 D - nDCG@k - [0.19242235]
10 G - nDCG@k - [0.6165324]
10 D - nDCG@k - [0.19242235]
11 G - nDCG@k - [0.6165324]
11 D - nDCG@k - [0.19242235]
12 G - nDCG@k - [0.6165324]
12 D - nDCG@k - [0.19242235]
13 G - nDCG@k - [0.6165324]
13 D - nDCG@k - [0.19242235]
14 G - nDCG@k - [0.6165324]
14 D - nDCG@k - [0.19242235]
15 G - nDCG@k - [0.6165324]
15 D - nDCG@k - [0.19242235]
16 G - nDCG@k - [0.6165324]
16 D - nDCG@k - [0.19242235]
17 G - nDCG@k - [0.6165324]
17 D - nDCG@k - [0.19242235]
18 G - nDCG@k - [0.6165324]
18 D - nDCG@k - [0.19242235]
19 G - nDCG@k - [0.6165324]
19 D - nDCG@k - [0.19242235]
20 G - nDCG@k - [0.6165324]
20 D - nDCG@k - [0.19242235]
21 G - nDCG@k - [0.6165324]
21 D - nDCG@k - [0.19242235]
22 G - nDCG@k - [0.6165324]
22 D - nDCG@k - [0.19242235]
23 G - nDCG@k - [0.6165324]
23 D - nDCG@k - [0.19242235]
24 G - nDCG@k - [0.6165324]
24 D - nDCG@k - [0.19242235]
25 G - nDCG@k - [0.6165324]
25 D - nDCG@k - [0.19242235]
26 G - nDCG@k - [0.6165324]
26 D - nDCG@k - [0.19242235]
27 G - nDCG@k - [0.6165324]
27 D - nDCG@k - [0.19242235]
28 G - nDCG@k - [0.6165324]
28 D - nDCG@k - [0.19242235]
29 G - nDCG@k - [0.6165324]
29 D - nDCG@k - [0.19242235]
30 G - nDCG@k - [0.6165324]
30 D - nDCG@k - [0.19242235]
31 G - nDCG@k - [0.6165324]
31 D - nDCG@k - [0.19242235]
32 G - nDCG@k - [0.6165324]
32 D - nDCG@k - [0.19242235]
33 G - nDCG@k - [0.6165324]
33 D - nDCG@k - [0.19242235]
34 G - nDCG@k - [0.6165324]
34 D - nDCG@k - [0.19242235]
35 G - nDCG@k - [0.6165324]
35 D - nDCG@k - [0.19242235]
36 G - nDCG@k - [0.6165324]
36 D - nDCG@k - [0.19242235]
37 G - nDCG@k - [0.6165324]
37 D - nDCG@k - [0.19242235]
38 G - nDCG@k - [0.6165324]
38 D - nDCG@k - [0.19242235]
39 G - nDCG@k - [0.6165324]
39 D - nDCG@k - [0.19242235]
40 G - nDCG@k - [0.6165324]
40 D - nDCG@k - [0.19242235]
41 G - nDCG@k - [0.6165324]
41 D - nDCG@k - [0.19242235]
42 G - nDCG@k - [0.6165324]
42 D - nDCG@k - [0.19242235]
43 G - nDCG@k - [0.6165324]
43 D - nDCG@k - [0.19242235]
44 G - nDCG@k - [0.6165324]
44 D - nDCG@k - [0.19242235]
45 G - nDCG@k - [0.6165324]
45 D - nDCG@k - [0.19242235]
46 G - nDCG@k - [0.6165324]
46 D - nDCG@k - [0.19242235]
47 G - nDCG@k - [0.6165324]
47 D - nDCG@k - [0.19242235]
48 G - nDCG@k - [0.6165324]
48 D - nDCG@k - [0.19242235]
49 G - nDCG@k - [0.6165324]
49 D - nDCG@k - [0.19242235]
50 G - nDCG@k - [0.6165324]
50 D - nDCG@k - [0.19242235]
Fold-1 Generator nDCG@1:0.3205 nDCG@3:0.5134 nDCG@5:0.5825 nDCG@10:0.6391 nDCG@20:0.2216 nDCG@50:0.1002
Discriminator nDCG@1:0.1090 nDCG@3:0.1455 nDCG@5:0.1578 nDCG@10:0.2289 nDCG@20:0.0824 nDCG@50:0.0562
Fold- 2
loading buffered file ...
loading buffered file ...
loading buffered file ...
2 G - nDCG@k - [0.5027733]
2 D - nDCG@k - [0.33297938]
3 G - nDCG@k - [0.48851907]
3 D - nDCG@k - [0.3581972]
4 G - nDCG@k - [0.49908563]
4 D - nDCG@k - [0.535753]
5 G - nDCG@k - [0.50161684]
5 D - nDCG@k - [0.5444428]
6 G - nDCG@k - [0.49681264]
6 D - nDCG@k - [0.5036262]
7 G - nDCG@k - [0.4993641]
7 D - nDCG@k - [0.54788077]
8 G - nDCG@k - [0.5029315]
8 D - nDCG@k - [0.505778]
9 G - nDCG@k - [0.5097145]
9 D - nDCG@k - [0.5421982]
10 G - nDCG@k - [0.5139733]
10 D - nDCG@k - [0.5388133]
11 G - nDCG@k - [0.5098361]
11 D - nDCG@k - [0.55477446]
12 G - nDCG@k - [0.5148136]
12 D - nDCG@k - [0.55512834]
13 G - nDCG@k - [0.5041314]
13 D - nDCG@k - [0.54865634]
14 G - nDCG@k - [0.5089696]
14 D - nDCG@k - [0.5308094]
15 G - nDCG@k - [0.517816]
15 D - nDCG@k - [0.49637938]
16 G - nDCG@k - [0.51141846]
16 D - nDCG@k - [0.54391533]
17 G - nDCG@k - [0.52058184]
17 D - nDCG@k - [0.519878]
18 G - nDCG@k - [0.5096874]
18 D - nDCG@k - [0.5619784]
19 G - nDCG@k - [0.5131619]
19 D - nDCG@k - [0.5138939]
20 G - nDCG@k - [0.51126856]
20 D - nDCG@k - [0.59303594]
21 G - nDCG@k - [0.5158795]
21 D - nDCG@k - [0.5766944]
22 G - nDCG@k - [0.50954753]
22 D - nDCG@k - [0.576388]
23 G - nDCG@k - [0.5153102]
23 D - nDCG@k - [0.5130749]
24 G - nDCG@k - [0.51462805]
24 D - nDCG@k - [0.5305879]
25 G - nDCG@k - [0.51734453]
25 D - nDCG@k - [0.56582767]
26 G - nDCG@k - [0.51586586]
26 D - nDCG@k - [0.5670248]
27 G - nDCG@k - [0.51664823]
27 D - nDCG@k - [0.5526367]
28 G - nDCG@k - [0.51834965]
28 D - nDCG@k - [0.4996301]
29 G - nDCG@k - [0.5147342]
29 D - nDCG@k - [0.5913757]
30 G - nDCG@k - [0.51855457]
30 D - nDCG@k - [0.5699038]
31 G - nDCG@k - [0.513338]
31 D - nDCG@k - [0.52834344]
32 G - nDCG@k - [0.5147203]
32 D - nDCG@k - [0.548412]
33 G - nDCG@k - [0.5150379]
33 D - nDCG@k - [0.57615685]
34 G - nDCG@k - [0.5167229]
34 D - nDCG@k - [0.57172066]
35 G - nDCG@k - [0.5218259]
35 D - nDCG@k - [0.56834656]
36 G - nDCG@k - [0.5258375]
36 D - nDCG@k - [0.57411903]
37 G - nDCG@k - [0.53553724]
37 D - nDCG@k - [0.54351085]
38 G - nDCG@k - [0.5644661]
38 D - nDCG@k - [0.5431185]
39 G - nDCG@k - [0.7448567]
39 D - nDCG@k - [0.550732]
40 G - nDCG@k - [0.94387144]
40 D - nDCG@k - [0.5618674]
41 G - nDCG@k - [1.]
41 D - nDCG@k - [0.5317064]
42 G - nDCG@k - [1.]
42 D - nDCG@k - [0.52337396]
43 G - nDCG@k - [1.]
43 D - nDCG@k - [0.5534222]
44 G - nDCG@k - [1.]
44 D - nDCG@k - [0.53518224]
45 G - nDCG@k - [1.]
45 D - nDCG@k - [0.56251013]
46 G - nDCG@k - [1.]
46 D - nDCG@k - [0.5596059]
47 G - nDCG@k - [1.]
47 D - nDCG@k - [0.58057314]
48 G - nDCG@k - [1.]
48 D - nDCG@k - [0.5857386]
49 G - nDCG@k - [1.]
49 D - nDCG@k - [0.56591916]
50 G - nDCG@k - [1.]
50 D - nDCG@k - [0.58940756]
Fold-2 Generator nDCG@1:1.0000 nDCG@3:1.0000 nDCG@5:1.0000 nDCG@10:1.0000 nDCG@20:0.6200 nDCG@50:0.2400
Discriminator nDCG@1:0.3333 nDCG@3:0.3821 nDCG@5:0.4599 nDCG@10:0.5160 nDCG@20:0.3420 nDCG@50:0.1332
Fold- 3
loading buffered file ...
loading buffered file ...
loading buffered file ...
2 G - nDCG@k - [0.39965856]
2 D - nDCG@k - [0.39463258]
3 G - nDCG@k - [0.3977977]
3 D - nDCG@k - [0.42421216]
4 G - nDCG@k - [0.38794923]
4 D - nDCG@k - [0.40071067]
5 G - nDCG@k - [0.38839263]
5 D - nDCG@k - [0.47236297]
6 G - nDCG@k - [0.38685566]
6 D - nDCG@k - [0.41886345]
7 G - nDCG@k - [0.38557106]
7 D - nDCG@k - [0.44698852]
8 G - nDCG@k - [0.38285]
8 D - nDCG@k - [0.48515868]
9 G - nDCG@k - [0.38673398]
9 D - nDCG@k - [0.42064533]
10 G - nDCG@k - [0.38321465]
10 D - nDCG@k - [0.41532105]
11 G - nDCG@k - [0.381612]
11 D - nDCG@k - [0.43045655]
12 G - nDCG@k - [0.38329402]
12 D - nDCG@k - [0.44125608]
13 G - nDCG@k - [0.3784583]
13 D - nDCG@k - [0.41501996]
14 G - nDCG@k - [0.372635]
14 D - nDCG@k - [0.3938498]
15 G - nDCG@k - [0.38326636]
15 D - nDCG@k - [0.45585468]
16 G - nDCG@k - [0.37885362]
16 D - nDCG@k - [0.42541683]
17 G - nDCG@k - [0.37920886]
17 D - nDCG@k - [0.4466286]
18 G - nDCG@k - [0.38100868]
18 D - nDCG@k - [0.4115614]
19 G - nDCG@k - [0.3843449]
19 D - nDCG@k - [0.42056996]
20 G - nDCG@k - [0.4572133]
20 D - nDCG@k - [0.4343956]
21 G - nDCG@k - [1.]
21 D - nDCG@k - [0.42924148]
22 G - nDCG@k - [1.]
22 D - nDCG@k - [0.43273562]
23 G - nDCG@k - [1.]
23 D - nDCG@k - [0.41558936]
24 G - nDCG@k - [1.]
24 D - nDCG@k - [0.42072624]
25 G - nDCG@k - [1.]
25 D - nDCG@k - [0.44657457]
26 G - nDCG@k - [1.]
26 D - nDCG@k - [0.45624048]
27 G - nDCG@k - [1.]
27 D - nDCG@k - [0.4384963]
28 G - nDCG@k - [1.]
28 D - nDCG@k - [0.44075084]
29 G - nDCG@k - [1.]
29 D - nDCG@k - [0.46042985]
30 G - nDCG@k - [1.]
30 D - nDCG@k - [0.41602588]
31 G - nDCG@k - [1.]
31 D - nDCG@k - [0.4504845]
32 G - nDCG@k - [1.]
32 D - nDCG@k - [0.45323014]
33 G - nDCG@k - [1.]
33 D - nDCG@k - [0.43309265]
34 G - nDCG@k - [1.]
34 D - nDCG@k - [0.46374264]
35 G - nDCG@k - [1.]
35 D - nDCG@k - [0.41332752]
36 G - nDCG@k - [1.]
36 D - nDCG@k - [0.4222603]
37 G - nDCG@k - [1.]
37 D - nDCG@k - [0.41125506]
38 G - nDCG@k - [1.]
38 D - nDCG@k - [0.4020129]
39 G - nDCG@k - [1.]
39 D - nDCG@k - [0.44501936]
40 G - nDCG@k - [1.]
40 D - nDCG@k - [0.4550639]
41 G - nDCG@k - [1.]
41 D - nDCG@k - [0.43464988]
42 G - nDCG@k - [1.]
42 D - nDCG@k - [0.414366]
43 G - nDCG@k - [1.]
43 D - nDCG@k - [0.42860523]
44 G - nDCG@k - [1.]
44 D - nDCG@k - [0.42242187]
45 G - nDCG@k - [1.]
45 D - nDCG@k - [0.43204117]
46 G - nDCG@k - [1.]
46 D - nDCG@k - [0.41403836]
47 G - nDCG@k - [1.]
47 D - nDCG@k - [0.37607488]
48 G - nDCG@k - [1.]
48 D - nDCG@k - [0.39542264]
49 G - nDCG@k - [1.]
49 D - nDCG@k - [0.41776687]
50 G - nDCG@k - [1.]
50 D - nDCG@k - [0.44091678]
Fold-3 Generator nDCG@1:1.0000 nDCG@3:1.0000 nDCG@5:1.0000 nDCG@10:1.0000 nDCG@20:0.6364 nDCG@50:0.3030
Discriminator nDCG@1:0.3737 nDCG@3:0.4729 nDCG@5:0.4982 nDCG@10:0.5583 nDCG@20:0.3616 nDCG@50:0.1863
Fold- 4
loading buffered file ...
loading buffered file ...
loading buffered file ...
2 G - nDCG@k - [0.47818083]
2 D - nDCG@k - [0.48583528]
3 G - nDCG@k - [0.47544459]
3 D - nDCG@k - [0.5115808]
4 G - nDCG@k - [0.47582328]
4 D - nDCG@k - [0.51140046]
5 G - nDCG@k - [0.4757267]
5 D - nDCG@k - [0.5173592]
6 G - nDCG@k - [0.4664649]
6 D - nDCG@k - [0.5153646]
7 G - nDCG@k - [0.4853018]
7 D - nDCG@k - [0.50688297]
8 G - nDCG@k - [0.47816467]
8 D - nDCG@k - [0.4853911]
9 G - nDCG@k - [0.48528868]
9 D - nDCG@k - [0.50741774]
10 G - nDCG@k - [0.49050823]
10 D - nDCG@k - [0.5098825]
11 G - nDCG@k - [0.48336774]
11 D - nDCG@k - [0.50937814]
12 G - nDCG@k - [0.5164186]
12 D - nDCG@k - [0.5006186]
13 G - nDCG@k - [1.]
13 D - nDCG@k - [0.50246394]
14 G - nDCG@k - [1.]
14 D - nDCG@k - [0.50000286]
15 G - nDCG@k - [1.]
15 D - nDCG@k - [0.47401243]
16 G - nDCG@k - [1.]
16 D - nDCG@k - [0.45529673]
17 G - nDCG@k - [1.]
17 D - nDCG@k - [0.4995319]
18 G - nDCG@k - [1.]
18 D - nDCG@k - [0.45995122]
19 G - nDCG@k - [1.]
19 D - nDCG@k - [0.5125517]
20 G - nDCG@k - [1.]
20 D - nDCG@k - [0.4958896]
21 G - nDCG@k - [1.]
21 D - nDCG@k - [0.5001597]
22 G - nDCG@k - [1.]
22 D - nDCG@k - [0.5115064]
23 G - nDCG@k - [1.]
23 D - nDCG@k - [0.49558964]
24 G - nDCG@k - [1.]
24 D - nDCG@k - [0.510579]
25 G - nDCG@k - [1.]
25 D - nDCG@k - [0.5273005]
26 G - nDCG@k - [1.]
26 D - nDCG@k - [0.49702188]
27 G - nDCG@k - [1.]
27 D - nDCG@k - [0.5037141]
28 G - nDCG@k - [1.]
28 D - nDCG@k - [0.5175707]
29 G - nDCG@k - [1.]
29 D - nDCG@k - [0.5381575]
30 G - nDCG@k - [1.]
30 D - nDCG@k - [0.47556967]
31 G - nDCG@k - [1.]
31 D - nDCG@k - [0.5207562]
32 G - nDCG@k - [1.]
32 D - nDCG@k - [0.5066564]
33 G - nDCG@k - [1.]
33 D - nDCG@k - [0.523348]
34 G - nDCG@k - [1.]
34 D - nDCG@k - [0.49611872]
35 G - nDCG@k - [1.]
35 D - nDCG@k - [0.51861805]
36 G - nDCG@k - [1.]
36 D - nDCG@k - [0.5162305]
37 G - nDCG@k - [1.]
37 D - nDCG@k - [0.5035715]
38 G - nDCG@k - [1.]
38 D - nDCG@k - [0.5117259]
39 G - nDCG@k - [1.]
39 D - nDCG@k - [0.51569456]
40 G - nDCG@k - [1.]
40 D - nDCG@k - [0.4981594]
41 G - nDCG@k - [1.]
41 D - nDCG@k - [0.50927716]
42 G - nDCG@k - [1.]
42 D - nDCG@k - [0.51942354]
43 G - nDCG@k - [1.]
43 D - nDCG@k - [0.4820416]
44 G - nDCG@k - [1.]
44 D - nDCG@k - [0.4894538]
45 G - nDCG@k - [1.]
45 D - nDCG@k - [0.4829564]
46 G - nDCG@k - [1.]
46 D - nDCG@k - [0.50496817]
47 G - nDCG@k - [1.]
47 D - nDCG@k - [0.497437]
48 G - nDCG@k - [1.]
48 D - nDCG@k - [0.51842123]
49 G - nDCG@k - [1.]
49 D - nDCG@k - [0.51879853]
50 G - nDCG@k - [1.]
50 D - nDCG@k - [0.45174882]
Fold-4 Generator nDCG@1:1.0000 nDCG@3:1.0000 nDCG@5:1.0000 nDCG@10:1.0000 nDCG@20:0.4928 nDCG@50:0.1884
Discriminator nDCG@1:0.5266 nDCG@3:0.5334 nDCG@5:0.5792 nDCG@10:0.6556 nDCG@20:0.2867 nDCG@50:0.1137
Fold- 5
loading buffered file ...
loading buffered file ...
loading buffered file ...
2 G - nDCG@k - [0.527702]
2 D - nDCG@k - [0.36692753]
3 G - nDCG@k - [0.5332169]
3 D - nDCG@k - [0.3692329]
4 G - nDCG@k - [0.53697956]
4 D - nDCG@k - [0.36051306]
5 G - nDCG@k - [0.5326443]
5 D - nDCG@k - [0.45014596]
6 G - nDCG@k - [0.5318162]
6 D - nDCG@k - [0.48854607]
7 G - nDCG@k - [0.5415791]
7 D - nDCG@k - [0.50873864]
8 G - nDCG@k - [0.5456936]
8 D - nDCG@k - [0.4943236]
9 G - nDCG@k - [0.5460599]
9 D - nDCG@k - [0.5161224]
10 G - nDCG@k - [0.5552072]
10 D - nDCG@k - [0.52876776]
11 G - nDCG@k - [0.55880207]
11 D - nDCG@k - [0.5287629]
12 G - nDCG@k - [0.5556395]
12 D - nDCG@k - [0.52552664]
13 G - nDCG@k - [0.5553135]
13 D - nDCG@k - [0.5456068]
14 G - nDCG@k - [0.55035204]
14 D - nDCG@k - [0.52752155]
15 G - nDCG@k - [0.5534008]
15 D - nDCG@k - [0.5242569]
16 G - nDCG@k - [0.55459607]
16 D - nDCG@k - [0.51693445]
17 G - nDCG@k - [0.56200135]
17 D - nDCG@k - [0.5370413]
18 G - nDCG@k - [0.5607125]
18 D - nDCG@k - [0.5368279]
19 G - nDCG@k - [0.5612805]
19 D - nDCG@k - [0.53559524]
20 G - nDCG@k - [0.56169796]
20 D - nDCG@k - [0.5444798]
21 G - nDCG@k - [0.5609834]
21 D - nDCG@k - [0.5295129]
22 G - nDCG@k - [0.5635907]
22 D - nDCG@k - [0.5265945]
23 G - nDCG@k - [0.557672]
23 D - nDCG@k - [0.54104066]
24 G - nDCG@k - [0.558078]
24 D - nDCG@k - [0.5258157]
25 G - nDCG@k - [0.5564844]
25 D - nDCG@k - [0.52085525]
26 G - nDCG@k - [0.5628345]
26 D - nDCG@k - [0.52485234]
27 G - nDCG@k - [0.5609715]
27 D - nDCG@k - [0.5222826]
28 G - nDCG@k - [0.56284297]
28 D - nDCG@k - [0.52649665]
29 G - nDCG@k - [0.5494826]
29 D - nDCG@k - [0.5414615]
30 G - nDCG@k - [0.5550785]
30 D - nDCG@k - [0.5354695]
31 G - nDCG@k - [0.552811]
31 D - nDCG@k - [0.5362496]
32 G - nDCG@k - [0.55090904]
32 D - nDCG@k - [0.5380133]
33 G - nDCG@k - [0.550326]
33 D - nDCG@k - [0.55514485]
34 G - nDCG@k - [0.5569996]
34 D - nDCG@k - [0.5370644]
35 G - nDCG@k - [0.55522406]
35 D - nDCG@k - [0.5436135]
36 G - nDCG@k - [0.553612]
36 D - nDCG@k - [0.53660625]
37 G - nDCG@k - [0.5529566]
37 D - nDCG@k - [0.5582198]
38 G - nDCG@k - [0.56821305]
38 D - nDCG@k - [0.55460787]
39 G - nDCG@k - [0.57327646]
39 D - nDCG@k - [0.54661214]
40 G - nDCG@k - [0.57777023]
40 D - nDCG@k - [0.5611896]
41 G - nDCG@k - [0.60631126]
41 D - nDCG@k - [0.5449414]
42 G - nDCG@k - [0.6949094]
42 D - nDCG@k - [0.5537747]
43 G - nDCG@k - [0.9749628]
43 D - nDCG@k - [0.5671416]
44 G - nDCG@k - [1.]
44 D - nDCG@k - [0.5487286]
45 G - nDCG@k - [1.]
45 D - nDCG@k - [0.5448432]
46 G - nDCG@k - [1.]
46 D - nDCG@k - [0.55738384]
47 G - nDCG@k - [1.]
47 D - nDCG@k - [0.5713886]
48 G - nDCG@k - [1.]
48 D - nDCG@k - [0.5563294]
49 G - nDCG@k - [1.]
49 D - nDCG@k - [0.5734414]
50 G - nDCG@k - [1.]
50 D - nDCG@k - [0.5654034]
Fold-5 Generator nDCG@1:1.0000 nDCG@3:1.0000 nDCG@5:1.0000 nDCG@10:1.0000 nDCG@20:0.4643 nDCG@50:0.1607
Discriminator nDCG@1:0.3690 nDCG@3:0.4526 nDCG@5:0.4893 nDCG@10:0.5792 nDCG@20:0.2791 nDCG@50:0.0978
Elapsed time: 0:11:22.062286

Generator 5-fold cross validation scores: nDCG@1:0.8641, nDCG@3:0.9027, nDCG@5:0.9165, nDCG@10:0.9278, nDCG@20:0.4870, nDCG@50:0.1985
Discriminator 5-fold cross validation scores: nDCG@1:0.3423, nDCG@3:0.3973, nDCG@5:0.4369, nDCG@10:0.5076, nDCG@20:0.2703, nDCG@50:0.1174

这个计算是否有问题，未包含batch_ideal_sorted_labels参数
def torch_precision_at_k(batch_sys_sorted_labels, k=None, gpu=False):
''' Precision at k

:param sys_sorted_labels: [batch_size, ranking_size] system's predicted ltr_adhoc of labels in a descending order
batch_size维指query,ranking_size维指查询下的文档
:param ks: cutoff values
:return: [batch_size, len(ks)]
'''
max_cutoff = batch_sys_sorted_labels.size(1)
used_cutoff = min(max_cutoff, k)

batch_sys_sorted_labels = batch_sys_sorted_labels[:, 0:used_cutoff]
'''
取前used_cutoff个排序文档的label
'''
batch_bi_sys_sorted_labels = torch.clamp(batch_sys_sorted_labels, min=0, max=1) # binary
'''
将排序文档的label 限幅到0-1之间
'''
batch_sys_cumsum_reles = torch.cumsum(batch_bi_sys_sorted_labels, dim=1)
'''
将排序文档的label 依次累加
'''

batch_ranks = (torch.arange(used_cutoff).type(torch.cuda.FloatTensor).expand_as(batch_sys_cumsum_reles) + 1.0) \
				if gpu else (torch.arange(used_cutoff).expand_as(batch_sys_cumsum_reles) + 1.0)

batch_sys_rankwise_precision = batch_sys_cumsum_reles / batch_ranks
batch_sys_p_at_k = batch_sys_rankwise_precision[:, used_cutoff-1:used_cutoff]
return batch_sys_p_at_k

def torch_precision_at_ks(batch_sys_sorted_labels, ks=None, gpu=False):
''' Precision at ks
:param sys_sorted_labels: [batch_size, ranking_size] system's predicted ltr_adhoc of labels in a descending order
:param ks: cutoff values
:return: [batch_size, len(ks)]
'''
valid_max_cutoff = batch_sys_sorted_labels.size(1)
need_padding = True if valid_max_cutoff < max(ks) else False
used_ks = [k for k in ks if k <= valid_max_cutoff] if need_padding else ks

max_cutoff = max(used_ks)
inds = torch.from_numpy(np.asarray(used_ks) - 1)

batch_sys_sorted_labels = batch_sys_sorted_labels[:, 0:max_cutoff]
batch_bi_sys_sorted_labels = torch.clamp(batch_sys_sorted_labels, min=0, max=1) # binary
batch_sys_cumsum_reles = torch.cumsum(batch_bi_sys_sorted_labels, dim=1)

batch_ranks = (torch.arange(max_cutoff).type(torch.cuda.FloatTensor).expand_as(batch_sys_cumsum_reles) + 1.0) if gpu \
										else (torch.arange(max_cutoff).expand_as(batch_sys_cumsum_reles) + 1.0)

batch_sys_rankwise_precision = batch_sys_cumsum_reles / batch_ranks
batch_sys_p_at_ks = batch_sys_rankwise_precision[:, inds]
if need_padding:
	padded_p_at_ks = torch.zeros(batch_sys_sorted_labels.size(0), len(ks))
	padded_p_at_ks[:, 0:len(used_ks)] = batch_sys_p_at_ks
	return padded_p_at_ks
else:
	return batch_sys_p_at_ks

二维码过期了，方便更新一下吗，谢谢。

When I executed the program with the following command, python -data MQ2008_List -dir_data /Users/gallup/study/tensorflow-LTR/data/MQ2008/Fold1 -dir_output /home/dl-box/WorkBench/CodeBench/Project_output/Out_L2R/Listwise/ -model LightGBMLambdaMART -dir_json /Users/gallup/study/ptranking/testing/ltr_tree/json/

I encountered an error：
lightgbm.basic.LightGBMError: Label 1330 is not less than the number of label mappings (31)

Start LightGBMLambdaMART on MQ2008_List >>>
Fold- 1
/Users/gallup/opt/anaconda3/envs/tf2py3/lib/python3.8/site-packages/lightgbm/engine.py:151: UserWarning: Found num_trees in params. Will use it instead of argument
warnings.warn("Found {} in params. Will use it instead of argument".format(alias))
[LightGBM] [Fatal] Label 1330 is not less than the number of label mappings (31)
Traceback (most recent call last):
File "/Users/gallup/study/ptranking/pt_ranking.py", line 85, in
evaluator.run(model_id=l2r_args.model, dir_json=l2r_args.dir_json, debug=False, config_with_json=True)
File "/Users/gallup/study/ptranking/ptranking/ltr_adhoc/eval/ltr.py", line 604, in run
self.grid_run(debug=debug, model_id=model_id, dir_json=dir_json)
File "/Users/gallup/study/ptranking/ptranking/ltr_tree/eval/ltr_tree.py", line 380, in grid_run
self.kfold_cv_eval(data_dict=data_dict, eval_dict=eval_dict, model_para_dict=model_para_dict)
File "/Users/gallup/study/ptranking/ptranking/ltr_tree/eval/ltr_tree.py", line 257, in kfold_cv_eval
y_test, group_test, y_pred = tree_ranker.run(fold_k=fold_k, file_train=file_train, file_vali=file_vali,
File "/Users/gallup/study/ptranking/ptranking/ltr_tree/lambdamart/lightgbm_lambdaMART.py", line 109, in run
lgbm_ranker = lgbm.train(params=self.lightgbm_para_dict, verbose_eval=10,
File "/Users/gallup/opt/anaconda3/envs/tf2py3/lib/python3.8/site-packages/lightgbm/engine.py", line 231, in train
booster = Booster(params=params, train_set=train_set)
File "/Users/gallup/opt/anaconda3/envs/tf2py3/lib/python3.8/site-packages/lightgbm/basic.py", line 1988, in init
_safe_call(_LIB.LGBM_BoosterCreate(
File "/Users/gallup/opt/anaconda3/envs/tf2py3/lib/python3.8/site-packages/lightgbm/basic.py", line 55, in _safe_call
raise LightGBMError(decode_string(_LIB.LGBM_GetLastError()))
lightgbm.basic.LightGBMError: Label 1330 is not less than the number of label mappings (31)

Process finished with exit code 1

Hello, i'm trying to run ptranking_demo_ltr.ipynb in windows 10 conda environment, and receiving error when trying to execute this part of code:

ranknet_losses, ranknet_train_ndcgs, ranknet_test_ndcgs = ltr_evaluator.naive_train(ranker=ranknet, eval_dict=eval_dict, train_data=train_data, test_data=test_data)

show_var(ranknet_losses, lbl='Training loss') # show the variation of loss
plt.legend()
plt.title('RankNet')

error:

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-19-0a2b685f404d> in <module>
----> 1 ranknet_losses, ranknet_train_ndcgs, ranknet_test_ndcgs = ltr_evaluator.naive_train(ranker=ranknet, eval_dict=eval_dict, train_data=train_data, test_data=test_data)
      2 
      3 show_var(ranknet_losses, lbl='Training loss') # show the variation of loss
      4 plt.legend()
      5 plt.title('RankNet')

  ....\ptranking\ltr_adhoc\eval\ltr.py in naive_train(self, ranker, eval_dict, train_data, test_data, vali_data)
    446             list_losses.append(np_epoch_loss)
    447 
--> 448             test_ndcg_ks = ndcg_at_ks(ranker=ranker, test_data=test_data, ks=cutoffs, multi_level_rele=True)
    449             np_test_ndcg_ks = test_ndcg_ks.data.numpy()
    450             list_test_ndcgs.append(np_test_ndcg_ks)

....ptranking\ltr_adhoc\eval\eval_utils.py in ndcg_at_ks(ranker, test_data, ks, multi_level_rele, batch_mode)
     75         ideal_sorted_labels, _ = torch.sort(std_labels, descending=True)
     76 
---> 77         ndcg_at_ks = torch_nDCG_at_ks(sys_sorted_labels=sys_sorted_labels, ideal_sorted_labels=ideal_sorted_labels, ks=ks, multi_level_rele=multi_level_rele)
     78         sum_ndcg_at_ks = torch.add(sum_ndcg_at_ks, ndcg_at_ks)
     79         cnt += 1

 ....adhoc_metric.py in torch_nDCG_at_ks(sys_sorted_labels, ideal_sorted_labels, ks, multi_level_rele)
    278         inds = torch.from_numpy(np.asarray(used_ks) - 1)
    279         sys_dited_cgs = torch_discounted_cumu_gain_at_ks(sys_sorted_labels, max_cutoff=max(used_ks), multi_level_rele=multi_level_rele)
--> 280         sys_dited_cg_at_ks = sys_dited_cgs[inds]  # get cumulative gains at specified rank positions
    281         ideal_dited_cgs = torch_discounted_cumu_gain_at_ks(ideal_sorted_labels, max_cutoff=max(used_ks), multi_level_rele=multi_level_rele)
    282         ideal_dited_cg_at_ks = ideal_dited_cgs[inds]

IndexError: tensors used as indices must be long, byte or bool tensors

The environment has pip version of Pytorch 1.6

I noticed that in the code for ListMLE:

def inner_train(self, batch_preds, batch_stds, **kwargs):
	# loss function of ListMLE
	batch_logcumsumexps = apply_LogCumsumExp(batch_preds)
	batch_loss = torch.sum(batch_logcumsumexps - batch_preds)
	self.optimizer.zero_grad()
	batch_loss.backward()
	self.optimizer.step()

	return batch_loss

batch_stds was not actually used... How would you calculate loss without labels?

null

Hello!

The JSON file on https://wildltr.github.io/ptranking/how_to_start/Configuration/ is not available. Would you pls share the JSON file?

Many thanks,
Charlotte

Hello again! I've posted here before (#18), but my explanation of the issue was not good enough, and as the issue was closed (and not solved), I had no other option than to submit a new issue.

What I'm trying to understand, is how to rank a "new" dataset, that wasn't used for training, validation or testing.
What I mean is that I can run the code, and I believe a ranking model is created (.pkl file), but that is based on the data that was used for training. What if I want to rank new data, that I can't even use for training because I don't know the "right" rank order? Or is this not possible with ptranking?

I hope I could explain myself better now, and thanks in advance!

Hi!
I've managed to run ptranking on the "MQ2008_List" dataset, and in the output folder (defined in EvalSetting), I have a "net_params_epoch_4.pkl" file, in each of the Fold folders.

However, I can't understand how to predict new data (that shouldn't be used for training), can someone help me?

Thank you very much in advance!
Pedro

Hi there, I was trying to run the "testing/ltr_adhoc/testing_ltr_adhoc.py" but got the following error. I tried to use inds.long() but it does not work. Any idea?
Many thanks!

Traceback (most recent call last):
File "F:/PythonCodes/ptranking/testing/ltr_adhoc/testing_ltr_adhoc.py", line 132, in
evaluator.run(debug=debug, model_id=model_id, sf_id=sf_id, grid_search=grid_search,
File "F:\PythonCodes\ptranking\ptranking\ltr_adhoc\eval\ltr.py", line 582, in run
self.point_run(debug=debug, model_id=model_id, sf_id=sf_id,
File "F:\PythonCodes\ptranking\ptranking\ltr_adhoc\eval\ltr.py", line 517, in point_run
self.kfold_cv_eval(data_dict=data_dict, eval_dict=eval_dict, model_para_dict=model_para_dict,
File "F:\PythonCodes\ptranking\ptranking\ltr_adhoc\eval\ltr.py", line 366, in kfold_cv_eval
cv_tape.fold_evaluation(model_id=model_id, ranker=ranker, test_data=test_data, max_label=max_label, fold_k=fold_k)
File "F:\PythonCodes\ptranking\ptranking\ltr_adhoc\eval\parameter.py", line 725, in fold_evaluation
ranker.adhoc_performance_at_ks(test_data=test_data, ks=self.cutoffs, device='cpu', max_label=max_label)
File "F:\PythonCodes\ptranking\ptranking\base\ranker.py", line 229, in adhoc_performance_at_ks
batch_ndcg_at_ks = torch_ndcg_at_ks(batch_predict_rankings=batch_predict_rankings,
File "F:\PythonCodes\ptranking\ptranking\metric\adhoc\adhoc_metric.py", line 280, in torch_ndcg_at_ks
batch_ideal_dcg_at_ks = batch_ideal_dcgs[:, inds]
IndexError: tensors used as indices must be long, byte or bool tensors

Process finished with exit code 1

I basically follow the exact step with your tutorial on /tutorial/ptranking_demo_ltr.ipynb file. I used the same dataset from LETOR 4.0 MQ2008 but got a KeyError in this line of code lambdarank_losses, lambdarank_train_ndcgs, lambdarank_test_ndcgs = ltr_evaluator.naive_train(ranker=lambdarank, eval_dict=eval_dict, train_data=train_data, test_data=test_data). It was fine with the RankNet though.
Error:

label_type = kwargs['label_type']
     32         assert LABEL_TYPE.MultiLabel == label_type
     33         assert 'presort' in kwargs and kwargs['presort'] is True  # aiming for direct usage of ideal ranking

KeyError: 'label_type'

I am new to this area and I really appreciate you for providing such nice code. I tried to reproduce the results reported in the paper on MQ2008.

1. For ListMLE, I ran the following script
   python ptl2r.py -data MQ2008_super -dir_data ./datasets/MQ/MQ2008/ -dir_output ./outputs/ -model ListMLE

and I get the following results in the log file

ListMLE 5-fold cross validation scores: nDCG@1:0.4326, nDCG@3:0.4574, nDCG@5:0.5023, nDCG@10:0.5824, nDCG@20:0.3046, nDCG@50:0.1339

while in the paper the results are nDCG@1:0.4097, nDCG@3:0.4456, nDCG@5:0.4831, nDCG@10:0.5666, which are lower than the reproduced results.

2. For ListNet, I ran the following script
   python ptl2r.py -data MQ2008_super -dir_data ./datasets/MQ/MQ2008/ -dir_output ./outputs/ -model ListNet

and I get the following results

ListNet 5-fold cross validation scores: nDCG@1:0.4605, nDCG@3:0.4839, nDCG@5:0.5248, nDCG@10:0.6062, nDCG@20:0.3198, nDCG@50:0.1400

while in the paper the results are nDCG@1:0.4173, nDCG@3:0.4476, nDCG@5:0.4878, nDCG@10:0.5752, which are much lower than the reproduced results.

3. For WassRank, I ran the following script
   python ptl2r.py -data MQ2008_super -dir_data ./datasets/MQ/MQ2008/ -dir_output ./outputs/ -model WassRank

and I get the following results

WassRank 5-fold cross validation scores: nDCG@1:0.4559, nDCG@3:0.4712, nDCG@5:0.5135, nDCG@10:0.5989, nDCG@20:0.3112, nDCG@50:0.1333

while in the paper the results are nDCG@1:0.4384, nDCG@3:0.4594, nDCG@5:0.4987, nDCG@10:0.5833, which are lower than the reproduced results. And the results for WassRank are even worse than ListNet.

I think I might got something wrong and would you please give me some advice on reproducing the results reported in the paper? Thanks for your patience.

Thanks for your excellent work~

Some papers say "lambdaRank" and "lambdaMart" are pairwise approach, because they come from the ranknet which is a pairwise method. In your paper, you put the two approaches into listwise. What's your opinion?

Hi,

How do I set the library to only use CPU when I run it in Jupyter Notebook I can only found instructions for the command line

Hi,
it is very hard to find source code; it took me quite some time because I initially just found the website.
Please add a clear link to the REPO + code.
Otherwise one might (as I did) that the repo is just for the website as it is AFAIK usually for a repo with name *.github.io.
I like the work, just want that everybody can see it.

Best Felix