MotivesExtractor/eval.py at master · urinieto/MotivesExtractor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
#!/usr/bin/env python
"""
Script to run the evaluations.

To run the script:
    ./eval.py annotations/ estimations/

Both directories must have the same file names.

#############

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__author__ = "Oriol Nieto"
__copyright__ = "Copyright 2013, Music and Audio Research Lab (MARL)"
__license__ = "GPL"
__version__ = "1.0"
__email__ = "oriol@nyu.edu"

import argparse
import glob
import logging
import mir_eval
import os
import time
import pandas as pd


def process(refdir, estdir):
    references = glob.glob(os.path.join(refdir, "*-polyphonic.txt"))
    estimations = glob.glob(os.path.join(estdir, "*.txt"))
    results = pd.DataFrame()
    for ref, est in zip(references, estimations):
        assert os.path.basename(ref) == os.path.basename(est)
        logging.info("Evaluating file: %s", est)
        ref_pat = mir_eval.io.load_patterns(ref)
        est_pat = mir_eval.io.load_patterns(est)

        res = {}
        res["Est_F"], res["Est_P"], res["Est_R"] = \
            mir_eval.pattern.establishment_FPR(ref_pat, est_pat)
        res["Occ.75_F"], res["Occ.75_P"], res["Occ.75_R"] = \
            mir_eval.pattern.occurrence_FPR(ref_pat, est_pat, thres=.75)
        res["ThreeLayer_F"], res["ThreeLayer_P"], res["ThreeLayer_R"] = \
            mir_eval.pattern.three_layer_FPR(ref_pat, est_pat)
        res["Occ.5_F"], res["Occ.5_P"], res["Occ.5_R"] = \
            mir_eval.pattern.occurrence_FPR(ref_pat, est_pat, thres=.5)
        res["Std_F"], res["Std_P"], res["Std_R"] = \
            mir_eval.pattern.standard_FPR(ref_pat, est_pat)
        results = results.append(res, ignore_index=True)

    logging.info("Results per piece:")
    print results
    logging.info("Average Results:")
    print results.mean()


def main():
    """Main function."""
    parser = argparse.ArgumentParser(description=
        "Evals the algorithm using mir_eval",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("refdir", action="store",
                        help="Directory with the annotations")
    parser.add_argument("estdir", action="store",
                        help="Directory with the estimations")

    args = parser.parse_args()
    start_time = time.time()

    # Setup the logger
    logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s',
                        level=logging.INFO)

    # Run the algorithm
    process(args.refdir, args.estdir)

    logging.info("Done! Took %.2f seconds." % (time.time() - start_time))

if __name__ == "__main__":
    main()